2024-11-17 03:14:19,950 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 03:14:19,963 main DEBUG Took 0.011063 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 03:14:19,963 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 03:14:19,964 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 03:14:19,965 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 03:14:19,966 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,973 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 03:14:19,989 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,991 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,992 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,994 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,995 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,997 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,998 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:19,999 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:19,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:20,000 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:20,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:20,001 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:20,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:20,002 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:20,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:20,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:20,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 03:14:20,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:20,004 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 03:14:20,005 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 03:14:20,006 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 03:14:20,008 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 03:14:20,008 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 03:14:20,010 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 03:14:20,010 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 03:14:20,018 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 03:14:20,020 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 03:14:20,022 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 03:14:20,023 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 03:14:20,023 main DEBUG createAppenders(={Console}) 2024-11-17 03:14:20,024 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 03:14:20,024 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 03:14:20,024 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 03:14:20,025 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 03:14:20,025 main DEBUG OutputStream closed 2024-11-17 03:14:20,025 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 03:14:20,026 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 03:14:20,026 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 03:14:20,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 03:14:20,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 03:14:20,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 03:14:20,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 03:14:20,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 03:14:20,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 03:14:20,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 03:14:20,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 03:14:20,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 03:14:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 03:14:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 03:14:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 03:14:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 03:14:20,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 03:14:20,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 03:14:20,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 03:14:20,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 03:14:20,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 03:14:20,108 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 03:14:20,108 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 03:14:20,108 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 03:14:20,109 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T03:14:20,354 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd 2024-11-17 03:14:20,357 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 03:14:20,357 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T03:14:20,365 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-17T03:14:20,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T03:14:20,402 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d, deleteOnExit=true 2024-11-17T03:14:20,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T03:14:20,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/test.cache.data in system properties and HBase conf 2024-11-17T03:14:20,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T03:14:20,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/hadoop.log.dir in system properties and HBase conf 2024-11-17T03:14:20,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T03:14:20,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T03:14:20,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T03:14:20,491 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T03:14:20,590 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T03:14:20,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T03:14:20,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T03:14:20,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T03:14:20,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T03:14:20,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T03:14:20,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T03:14:20,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T03:14:20,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T03:14:20,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T03:14:20,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/nfs.dump.dir in system properties and HBase conf 2024-11-17T03:14:20,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/java.io.tmpdir in system properties and HBase conf 2024-11-17T03:14:20,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T03:14:20,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T03:14:20,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T03:14:21,822 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T03:14:21,898 INFO [Time-limited test {}] log.Log(170): Logging initialized @2736ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T03:14:21,971 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T03:14:22,035 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T03:14:22,063 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T03:14:22,063 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T03:14:22,065 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T03:14:22,079 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T03:14:22,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/hadoop.log.dir/,AVAILABLE} 2024-11-17T03:14:22,084 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T03:14:22,301 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/java.io.tmpdir/jetty-localhost-34591-hadoop-hdfs-3_4_1-tests_jar-_-any-5436546102577836088/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T03:14:22,309 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:34591} 2024-11-17T03:14:22,310 INFO [Time-limited test {}] server.Server(415): Started @3149ms 2024-11-17T03:14:22,935 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T03:14:22,943 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T03:14:22,945 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T03:14:22,945 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T03:14:22,946 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T03:14:22,946 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/hadoop.log.dir/,AVAILABLE} 2024-11-17T03:14:22,947 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T03:14:23,050 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/java.io.tmpdir/jetty-localhost-45109-hadoop-hdfs-3_4_1-tests_jar-_-any-13636459398087354250/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T03:14:23,051 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:45109} 2024-11-17T03:14:23,051 INFO [Time-limited test {}] server.Server(415): Started @3890ms 2024-11-17T03:14:23,102 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T03:14:24,211 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/data/data2/current/BP-617005350-172.17.0.3-1731813261238/current, will proceed with Du for space computation calculation, 2024-11-17T03:14:24,211 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/data/data1/current/BP-617005350-172.17.0.3-1731813261238/current, will proceed with Du for space computation calculation, 2024-11-17T03:14:24,256 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T03:14:24,320 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3402315d7aa11007 with lease ID 0xcfb7a477ccc91c75: Processing first storage report for DS-372a1302-cd72-4906-8fdd-8feeb7b57587 from datanode DatanodeRegistration(127.0.0.1:46721, datanodeUuid=6e400762-c501-488a-ad00-d12d75c2fc6e, infoPort=37705, infoSecurePort=0, ipcPort=39487, storageInfo=lv=-57;cid=testClusterID;nsid=1468859919;c=1731813261238) 2024-11-17T03:14:24,322 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3402315d7aa11007 with lease ID 0xcfb7a477ccc91c75: from storage DS-372a1302-cd72-4906-8fdd-8feeb7b57587 node DatanodeRegistration(127.0.0.1:46721, datanodeUuid=6e400762-c501-488a-ad00-d12d75c2fc6e, infoPort=37705, infoSecurePort=0, ipcPort=39487, storageInfo=lv=-57;cid=testClusterID;nsid=1468859919;c=1731813261238), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T03:14:24,323 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3402315d7aa11007 with lease ID 0xcfb7a477ccc91c75: Processing first storage report for DS-19a7c24f-1223-46e2-ae7a-7be3dc3b23e5 from datanode DatanodeRegistration(127.0.0.1:46721, datanodeUuid=6e400762-c501-488a-ad00-d12d75c2fc6e, infoPort=37705, infoSecurePort=0, ipcPort=39487, storageInfo=lv=-57;cid=testClusterID;nsid=1468859919;c=1731813261238) 2024-11-17T03:14:24,324 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3402315d7aa11007 with lease ID 0xcfb7a477ccc91c75: from storage DS-19a7c24f-1223-46e2-ae7a-7be3dc3b23e5 node DatanodeRegistration(127.0.0.1:46721, datanodeUuid=6e400762-c501-488a-ad00-d12d75c2fc6e, infoPort=37705, infoSecurePort=0, ipcPort=39487, storageInfo=lv=-57;cid=testClusterID;nsid=1468859919;c=1731813261238), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T03:14:24,356 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd 2024-11-17T03:14:24,430 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/zookeeper_0, clientPort=60535, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T03:14:24,439 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60535 2024-11-17T03:14:24,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:24,452 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:24,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741825_1001 (size=7) 2024-11-17T03:14:25,117 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 with version=8 2024-11-17T03:14:25,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/hbase-staging 2024-11-17T03:14:25,209 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T03:14:25,449 INFO [Time-limited test {}] client.ConnectionUtils(128): master/695c90f4bb18:0 server-side Connection retries=45 2024-11-17T03:14:25,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T03:14:25,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T03:14:25,462 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T03:14:25,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T03:14:25,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T03:14:25,604 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T03:14:25,666 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T03:14:25,676 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T03:14:25,680 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T03:14:25,707 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4796 (auto-detected) 2024-11-17T03:14:25,709 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-17T03:14:25,732 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42091 2024-11-17T03:14:25,754 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42091 connecting to ZooKeeper ensemble=127.0.0.1:60535 2024-11-17T03:14:25,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420910x0, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T03:14:25,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42091-0x10146c947c80000 connected 2024-11-17T03:14:26,259 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:26,261 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:26,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T03:14:26,272 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64, hbase.cluster.distributed=false 2024-11-17T03:14:26,296 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T03:14:26,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42091 2024-11-17T03:14:26,301 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42091 2024-11-17T03:14:26,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42091 2024-11-17T03:14:26,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42091 2024-11-17T03:14:26,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42091 2024-11-17T03:14:26,420 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/695c90f4bb18:0 server-side Connection retries=45 2024-11-17T03:14:26,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T03:14:26,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T03:14:26,423 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T03:14:26,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T03:14:26,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T03:14:26,426 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T03:14:26,428 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T03:14:26,429 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38753 2024-11-17T03:14:26,431 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38753 connecting to ZooKeeper ensemble=127.0.0.1:60535 2024-11-17T03:14:26,433 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:26,436 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:26,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387530x0, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T03:14:26,489 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38753-0x10146c947c80001 connected 2024-11-17T03:14:26,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387530x0, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T03:14:26,493 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T03:14:26,502 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T03:14:26,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T03:14:26,511 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T03:14:26,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38753 2024-11-17T03:14:26,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38753 2024-11-17T03:14:26,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38753 2024-11-17T03:14:26,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38753 2024-11-17T03:14:26,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38753 2024-11-17T03:14:26,529 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;695c90f4bb18:42091 2024-11-17T03:14:26,530 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/695c90f4bb18,42091,1731813265296 2024-11-17T03:14:26,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T03:14:26,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T03:14:26,594 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/695c90f4bb18,42091,1731813265296 2024-11-17T03:14:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T03:14:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:26,678 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T03:14:26,680 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/695c90f4bb18,42091,1731813265296 from backup master directory 2024-11-17T03:14:26,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/695c90f4bb18,42091,1731813265296 2024-11-17T03:14:26,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T03:14:26,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T03:14:26,732 WARN [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T03:14:26,733 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=695c90f4bb18,42091,1731813265296 2024-11-17T03:14:26,736 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T03:14:26,738 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T03:14:26,789 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/hbase.id] with ID: 2d5edb17-01db-469a-ad88-13203f32e6ee 2024-11-17T03:14:26,790 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/.tmp/hbase.id 2024-11-17T03:14:26,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741826_1002 (size=42) 2024-11-17T03:14:27,202 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/.tmp/hbase.id]:[hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/hbase.id] 2024-11-17T03:14:27,251 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T03:14:27,256 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T03:14:27,275 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-17T03:14:27,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:27,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:27,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741827_1003 (size=196) 2024-11-17T03:14:27,753 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:14:27,755 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T03:14:27,769 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:27,772 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T03:14:27,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741828_1004 (size=1189) 2024-11-17T03:14:28,225 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store 2024-11-17T03:14:28,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741829_1005 (size=34) 2024-11-17T03:14:28,648 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T03:14:28,651 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:14:28,652 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T03:14:28,653 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:14:28,653 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:14:28,654 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T03:14:28,655 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:14:28,655 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:14:28,656 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731813268652Disabling compacts and flushes for region at 1731813268652Disabling writes for close at 1731813268654 (+2 ms)Writing region close event to WAL at 1731813268655 (+1 ms)Closed at 1731813268655 2024-11-17T03:14:28,658 WARN [master/695c90f4bb18:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/.initializing 2024-11-17T03:14:28,658 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/WALs/695c90f4bb18,42091,1731813265296 2024-11-17T03:14:28,666 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T03:14:28,681 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=695c90f4bb18%2C42091%2C1731813265296, suffix=, logDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/WALs/695c90f4bb18,42091,1731813265296, archiveDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/oldWALs, maxLogs=10 2024-11-17T03:14:28,712 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/WALs/695c90f4bb18,42091,1731813265296/695c90f4bb18%2C42091%2C1731813265296.1731813268687, exclude list is [], retry=0 2024-11-17T03:14:28,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46721,DS-372a1302-cd72-4906-8fdd-8feeb7b57587,DISK] 2024-11-17T03:14:28,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T03:14:28,770 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/WALs/695c90f4bb18,42091,1731813265296/695c90f4bb18%2C42091%2C1731813265296.1731813268687 2024-11-17T03:14:28,772 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705)] 2024-11-17T03:14:28,772 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:14:28,773 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:14:28,777 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,778 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,850 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T03:14:28,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:28,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:28,858 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T03:14:28,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:28,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:14:28,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T03:14:28,867 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:28,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:14:28,868 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T03:14:28,872 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:28,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:14:28,873 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,877 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,878 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,884 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,885 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,889 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T03:14:28,894 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T03:14:28,899 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:14:28,901 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64222288, jitterRate=-0.04301333427429199}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T03:14:28,910 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731813268790Initializing all the Stores at 1731813268793 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813268794 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813268795 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813268795Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813268795Cleaning up temporary data from old regions at 1731813268885 (+90 ms)Region opened successfully at 1731813268909 (+24 ms) 2024-11-17T03:14:28,912 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T03:14:28,953 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a02b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=695c90f4bb18/172.17.0.3:0 2024-11-17T03:14:28,998 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T03:14:29,013 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T03:14:29,014 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T03:14:29,017 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T03:14:29,018 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T03:14:29,023 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-17T03:14:29,023 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T03:14:29,052 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T03:14:29,062 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T03:14:29,083 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T03:14:29,087 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T03:14:29,090 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T03:14:29,099 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T03:14:29,102 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T03:14:29,110 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T03:14:29,120 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T03:14:29,122 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T03:14:29,136 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T03:14:29,153 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T03:14:29,162 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T03:14:29,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T03:14:29,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T03:14:29,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:29,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:29,175 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=695c90f4bb18,42091,1731813265296, sessionid=0x10146c947c80000, setting cluster-up flag (Was=false) 2024-11-17T03:14:29,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:29,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:29,241 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T03:14:29,247 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=695c90f4bb18,42091,1731813265296 2024-11-17T03:14:29,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:29,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:29,304 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T03:14:29,310 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=695c90f4bb18,42091,1731813265296 2024-11-17T03:14:29,320 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T03:14:29,384 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T03:14:29,393 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T03:14:29,399 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T03:14:29,404 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 695c90f4bb18,42091,1731813265296 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T03:14:29,413 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/695c90f4bb18:0, corePoolSize=5, maxPoolSize=5 2024-11-17T03:14:29,413 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/695c90f4bb18:0, corePoolSize=5, maxPoolSize=5 2024-11-17T03:14:29,413 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/695c90f4bb18:0, corePoolSize=5, maxPoolSize=5 2024-11-17T03:14:29,413 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/695c90f4bb18:0, corePoolSize=5, maxPoolSize=5 2024-11-17T03:14:29,414 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/695c90f4bb18:0, corePoolSize=10, maxPoolSize=10 2024-11-17T03:14:29,414 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,414 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/695c90f4bb18:0, corePoolSize=2, maxPoolSize=2 2024-11-17T03:14:29,414 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,416 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731813299416 2024-11-17T03:14:29,418 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T03:14:29,419 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T03:14:29,420 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(746): ClusterId : 2d5edb17-01db-469a-ad88-13203f32e6ee 2024-11-17T03:14:29,420 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T03:14:29,421 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T03:14:29,423 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T03:14:29,423 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T03:14:29,424 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T03:14:29,424 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T03:14:29,424 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T03:14:29,425 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,428 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:29,428 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T03:14:29,428 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T03:14:29,430 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T03:14:29,430 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T03:14:29,432 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T03:14:29,433 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T03:14:29,434 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/695c90f4bb18:0:becomeActiveMaster-HFileCleaner.large.0-1731813269434,5,FailOnTimeoutGroup] 2024-11-17T03:14:29,435 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/695c90f4bb18:0:becomeActiveMaster-HFileCleaner.small.0-1731813269435,5,FailOnTimeoutGroup] 2024-11-17T03:14:29,436 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,436 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T03:14:29,437 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,438 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,438 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T03:14:29,438 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T03:14:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741831_1007 (size=1321) 2024-11-17T03:14:29,448 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T03:14:29,448 DEBUG [RS:0;695c90f4bb18:38753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c47ac69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=695c90f4bb18/172.17.0.3:0 2024-11-17T03:14:29,462 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;695c90f4bb18:38753 2024-11-17T03:14:29,465 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T03:14:29,466 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T03:14:29,466 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T03:14:29,468 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(2659): reportForDuty to master=695c90f4bb18,42091,1731813265296 with port=38753, startcode=1731813266381 2024-11-17T03:14:29,479 DEBUG [RS:0;695c90f4bb18:38753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T03:14:29,513 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46075, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T03:14:29,519 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 695c90f4bb18,38753,1731813266381 2024-11-17T03:14:29,521 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42091 {}] master.ServerManager(517): Registering regionserver=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:29,535 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:14:29,535 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39789 2024-11-17T03:14:29,535 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T03:14:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T03:14:29,547 DEBUG [RS:0;695c90f4bb18:38753 {}] zookeeper.ZKUtil(111): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/695c90f4bb18,38753,1731813266381 2024-11-17T03:14:29,547 WARN [RS:0;695c90f4bb18:38753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T03:14:29,547 INFO [RS:0;695c90f4bb18:38753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T03:14:29,547 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381 2024-11-17T03:14:29,549 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [695c90f4bb18,38753,1731813266381] 2024-11-17T03:14:29,575 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T03:14:29,590 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T03:14:29,594 INFO [RS:0;695c90f4bb18:38753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T03:14:29,595 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,596 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T03:14:29,601 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T03:14:29,602 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,602 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,602 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,602 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,603 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,603 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,603 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/695c90f4bb18:0, corePoolSize=2, maxPoolSize=2 2024-11-17T03:14:29,603 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,603 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,603 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,604 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,604 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,604 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/695c90f4bb18:0, corePoolSize=1, maxPoolSize=1 2024-11-17T03:14:29,604 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/695c90f4bb18:0, corePoolSize=3, maxPoolSize=3 2024-11-17T03:14:29,604 DEBUG [RS:0;695c90f4bb18:38753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0, corePoolSize=3, maxPoolSize=3 2024-11-17T03:14:29,605 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,605 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,605 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,606 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,606 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,606 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,38753,1731813266381-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T03:14:29,625 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T03:14:29,627 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,38753,1731813266381-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,627 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,627 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.Replication(171): 695c90f4bb18,38753,1731813266381 started 2024-11-17T03:14:29,645 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:29,645 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1482): Serving as 695c90f4bb18,38753,1731813266381, RpcServer on 695c90f4bb18/172.17.0.3:38753, sessionid=0x10146c947c80001 2024-11-17T03:14:29,646 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T03:14:29,646 DEBUG [RS:0;695c90f4bb18:38753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 695c90f4bb18,38753,1731813266381 2024-11-17T03:14:29,647 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '695c90f4bb18,38753,1731813266381' 2024-11-17T03:14:29,647 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T03:14:29,648 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T03:14:29,649 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T03:14:29,649 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T03:14:29,649 DEBUG [RS:0;695c90f4bb18:38753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 695c90f4bb18,38753,1731813266381 2024-11-17T03:14:29,649 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '695c90f4bb18,38753,1731813266381' 2024-11-17T03:14:29,649 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T03:14:29,650 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T03:14:29,650 DEBUG [RS:0;695c90f4bb18:38753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T03:14:29,650 INFO [RS:0;695c90f4bb18:38753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T03:14:29,651 INFO [RS:0;695c90f4bb18:38753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T03:14:29,755 INFO [RS:0;695c90f4bb18:38753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T03:14:29,758 INFO [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=695c90f4bb18%2C38753%2C1731813266381, suffix=, logDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381, archiveDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/oldWALs, maxLogs=32 2024-11-17T03:14:29,773 DEBUG [RS:0;695c90f4bb18:38753 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381/695c90f4bb18%2C38753%2C1731813266381.1731813269760, exclude list is [], retry=0 2024-11-17T03:14:29,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46721,DS-372a1302-cd72-4906-8fdd-8feeb7b57587,DISK] 2024-11-17T03:14:29,781 INFO [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381/695c90f4bb18%2C38753%2C1731813266381.1731813269760 2024-11-17T03:14:29,781 DEBUG [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705)] 2024-11-17T03:14:29,842 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T03:14:29,843 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:14:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741833_1009 (size=32) 2024-11-17T03:14:30,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:14:30,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T03:14:30,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T03:14:30,271 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T03:14:30,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T03:14:30,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T03:14:30,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T03:14:30,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T03:14:30,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T03:14:30,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T03:14:30,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740 2024-11-17T03:14:30,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740 2024-11-17T03:14:30,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T03:14:30,290 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T03:14:30,291 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:14:30,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T03:14:30,297 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:14:30,298 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64095873, jitterRate=-0.044897064566612244}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:14:30,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731813270261Initializing all the Stores at 1731813270266 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813270267 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813270267Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813270267Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813270267Cleaning up temporary data from old regions at 1731813270290 (+23 ms)Region opened successfully at 1731813270301 (+11 ms) 2024-11-17T03:14:30,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T03:14:30,301 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T03:14:30,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T03:14:30,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T03:14:30,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T03:14:30,303 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T03:14:30,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731813270301Disabling compacts and flushes for region at 1731813270301Disabling writes for close at 1731813270301Writing region close event to WAL at 1731813270302 (+1 ms)Closed at 1731813270302 2024-11-17T03:14:30,306 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T03:14:30,306 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T03:14:30,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T03:14:30,320 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T03:14:30,323 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T03:14:30,480 DEBUG [695c90f4bb18:42091 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T03:14:30,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:30,498 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 695c90f4bb18,38753,1731813266381, state=OPENING 2024-11-17T03:14:30,568 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T03:14:30,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:30,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:14:30,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T03:14:30,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T03:14:30,584 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T03:14:30,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:14:30,770 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T03:14:30,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38567, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T03:14:30,784 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T03:14:30,784 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T03:14:30,785 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T03:14:30,788 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=695c90f4bb18%2C38753%2C1731813266381.meta, suffix=.meta, logDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381, archiveDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/oldWALs, maxLogs=32 2024-11-17T03:14:30,803 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381/695c90f4bb18%2C38753%2C1731813266381.meta.1731813270790.meta, exclude list is [], retry=0 2024-11-17T03:14:30,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46721,DS-372a1302-cd72-4906-8fdd-8feeb7b57587,DISK] 2024-11-17T03:14:30,809 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/WALs/695c90f4bb18,38753,1731813266381/695c90f4bb18%2C38753%2C1731813266381.meta.1731813270790.meta 2024-11-17T03:14:30,809 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37705:37705)] 2024-11-17T03:14:30,810 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:14:30,811 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T03:14:30,814 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T03:14:30,818 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T03:14:30,821 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T03:14:30,822 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:14:30,822 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T03:14:30,823 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T03:14:30,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T03:14:30,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T03:14:30,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T03:14:30,830 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T03:14:30,830 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T03:14:30,833 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T03:14:30,833 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T03:14:30,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T03:14:30,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:30,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T03:14:30,837 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T03:14:30,838 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740 2024-11-17T03:14:30,841 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740 2024-11-17T03:14:30,844 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T03:14:30,844 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T03:14:30,845 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:14:30,848 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T03:14:30,850 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69867575, jitterRate=0.04110799729824066}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:14:30,850 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T03:14:30,851 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731813270823Writing region info on filesystem at 1731813270823Initializing all the Stores at 1731813270825 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813270825Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813270826 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813270826Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731813270826Cleaning up temporary data from old regions at 1731813270844 (+18 ms)Running coprocessor post-open hooks at 1731813270850 (+6 ms)Region opened successfully at 1731813270851 (+1 ms) 2024-11-17T03:14:30,858 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731813270761 2024-11-17T03:14:30,870 DEBUG [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T03:14:30,870 INFO [RS_OPEN_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T03:14:30,872 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:30,874 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 695c90f4bb18,38753,1731813266381, state=OPEN 2024-11-17T03:14:30,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T03:14:30,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T03:14:30,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T03:14:30,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T03:14:30,938 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:30,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T03:14:30,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=695c90f4bb18,38753,1731813266381 in 351 msec 2024-11-17T03:14:30,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T03:14:30,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 634 msec 2024-11-17T03:14:30,954 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T03:14:30,954 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T03:14:30,977 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:30,978 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:30,997 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:31,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39841, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:31,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6720 sec 2024-11-17T03:14:31,020 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731813271020, completionTime=-1 2024-11-17T03:14:31,022 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T03:14:31,022 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-17T03:14:31,047 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-17T03:14:31,047 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731813331047 2024-11-17T03:14:31,048 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731813391048 2024-11-17T03:14:31,048 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 25 msec 2024-11-17T03:14:31,050 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,42091,1731813265296-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,050 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,42091,1731813265296-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,051 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,42091,1731813265296-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,052 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-695c90f4bb18:42091, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,053 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,053 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,058 DEBUG [master/695c90f4bb18:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T03:14:31,078 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.345sec 2024-11-17T03:14:31,079 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T03:14:31,080 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T03:14:31,081 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T03:14:31,082 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T03:14:31,082 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T03:14:31,083 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,42091,1731813265296-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T03:14:31,083 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,42091,1731813265296-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T03:14:31,093 DEBUG [master/695c90f4bb18:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T03:14:31,094 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T03:14:31,095 INFO [master/695c90f4bb18:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=695c90f4bb18,42091,1731813265296-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T03:14:31,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:31,137 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T03:14:31,137 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T03:14:31,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:31,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:31,149 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:31,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:31,160 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:31,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:31,163 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:31,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:31,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=695c90f4bb18,42091,1731813265296 2024-11-17T03:14:31,204 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=1941 2024-11-17T03:14:31,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:14:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:14:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-17T03:14:31,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:14:31,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:14:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:14:31,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:14:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T03:14:31,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T03:14:31,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T03:14:31,260 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:31,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T03:14:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T03:14:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741835_1011 (size=1061) 2024-11-17T03:14:31,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T03:14:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T03:14:31,688 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:14:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741836_1012 (size=53) 2024-11-17T03:14:31,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T03:14:32,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:14:32,101 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b29e9d27b06f8df48f1da9682bc65b77, disabling compactions & flushes 2024-11-17T03:14:32,101 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. after waiting 0 ms 2024-11-17T03:14:32,102 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,102 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b29e9d27b06f8df48f1da9682bc65b77: Waiting for close lock at 1731813272101Disabling compacts and flushes for region at 1731813272101Disabling writes for close at 1731813272102 (+1 ms)Writing region close event to WAL at 1731813272102Closed at 1731813272102 2024-11-17T03:14:32,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T03:14:32,112 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731813272106"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731813272106"}]},"ts":"1731813272106"} 2024-11-17T03:14:32,117 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T03:14:32,119 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T03:14:32,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813272119"}]},"ts":"1731813272119"} 2024-11-17T03:14:32,128 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T03:14:32,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, ASSIGN}] 2024-11-17T03:14:32,132 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, ASSIGN 2024-11-17T03:14:32,134 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, ASSIGN; state=OFFLINE, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=false 2024-11-17T03:14:32,287 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b29e9d27b06f8df48f1da9682bc65b77, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:32,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, ASSIGN because future has completed 2024-11-17T03:14:32,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:14:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T03:14:32,458 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,458 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:14:32,459 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,459 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:14:32,459 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,460 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,463 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,467 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:14:32,467 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b29e9d27b06f8df48f1da9682bc65b77 columnFamilyName A 2024-11-17T03:14:32,467 DEBUG [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:32,469 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.HStore(327): Store=b29e9d27b06f8df48f1da9682bc65b77/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:14:32,469 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,473 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:14:32,474 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b29e9d27b06f8df48f1da9682bc65b77 columnFamilyName B 2024-11-17T03:14:32,474 DEBUG [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:32,475 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.HStore(327): Store=b29e9d27b06f8df48f1da9682bc65b77/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:14:32,475 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,477 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:14:32,477 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b29e9d27b06f8df48f1da9682bc65b77 columnFamilyName C 2024-11-17T03:14:32,477 DEBUG [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:14:32,478 INFO [StoreOpener-b29e9d27b06f8df48f1da9682bc65b77-1 {}] regionserver.HStore(327): Store=b29e9d27b06f8df48f1da9682bc65b77/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:14:32,479 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,479 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,480 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,481 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,483 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,483 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,484 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:14:32,486 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,490 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:14:32,491 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b29e9d27b06f8df48f1da9682bc65b77; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74141591, jitterRate=0.10479579865932465}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:14:32,491 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:32,492 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b29e9d27b06f8df48f1da9682bc65b77: Running coprocessor pre-open hook at 1731813272460Writing region info on filesystem at 1731813272460Initializing all the Stores at 1731813272462 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813272462Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813272462Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813272462Cleaning up temporary data from old regions at 1731813272483 (+21 ms)Running coprocessor post-open hooks at 1731813272491 (+8 ms)Region opened successfully at 1731813272492 (+1 ms) 2024-11-17T03:14:32,494 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., pid=6, masterSystemTime=1731813272450 2024-11-17T03:14:32,498 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,498 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:32,499 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b29e9d27b06f8df48f1da9682bc65b77, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:32,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:14:32,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T03:14:32,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 in 208 msec 2024-11-17T03:14:32,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T03:14:32,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, ASSIGN in 380 msec 2024-11-17T03:14:32,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T03:14:32,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813272514"}]},"ts":"1731813272514"} 2024-11-17T03:14:32,518 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T03:14:32,519 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T03:14:32,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2710 sec 2024-11-17T03:14:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T03:14:33,419 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:33,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecb3032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,434 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3922c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,436 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,443 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:14:33,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:14:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7eef39e4 2024-11-17T03:14:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:14:33,449 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:14:33,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,455 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a183203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,458 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,461 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1bd73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,464 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,468 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,476 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,479 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eefee7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,481 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,482 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345c59db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,485 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,490 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,492 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1455c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,496 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,498 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4501fb62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,502 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3b865c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339e1ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:14:33,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:14:33,508 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:14:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:14:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dcce6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:33,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:14:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9c26bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:14:33,516 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,516 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,516 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:33,517 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,517 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,519 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,520 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,520 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,520 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,522 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,523 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T03:14:33,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T03:14:33,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,528 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:33,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,548 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,552 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:14:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:14:33,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:14:33,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:14:33,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:14:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:33,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:14:33,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:33,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T03:14:33,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:33,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:33,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:33,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:33,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T03:14:33,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:33,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:33,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:33,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813333716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813333725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813333726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813333731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:33,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813333735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/f6de317de0db41279fe4a50e7d47a621 is 50, key is test_row_0/A:col10/1731813273613/Put/seqid=0 2024-11-17T03:14:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741837_1013 (size=12001) 2024-11-17T03:14:33,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/f6de317de0db41279fe4a50e7d47a621 2024-11-17T03:14:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T03:14:33,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813333888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/096eb8198cfe4610a81ab398c589f594 is 50, key is test_row_0/B:col10/1731813273613/Put/seqid=0 2024-11-17T03:14:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813333888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813333889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813333890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813333890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:33,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T03:14:33,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:33,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:33,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:33,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741838_1014 (size=12001) 2024-11-17T03:14:34,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T03:14:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:34,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813334099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813334108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813334108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813334109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813334109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T03:14:34,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T03:14:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/096eb8198cfe4610a81ab398c589f594 2024-11-17T03:14:34,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/1e9e667b874b4b278fb6259d81ee73a9 is 50, key is test_row_0/C:col10/1731813273613/Put/seqid=0 2024-11-17T03:14:34,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T03:14:34,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:34,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:34,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741839_1015 (size=12001) 2024-11-17T03:14:34,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/1e9e667b874b4b278fb6259d81ee73a9 2024-11-17T03:14:34,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813334408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813334418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813334418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/f6de317de0db41279fe4a50e7d47a621 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f6de317de0db41279fe4a50e7d47a621 2024-11-17T03:14:34,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813334428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813334428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:34,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f6de317de0db41279fe4a50e7d47a621, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:14:34,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/096eb8198cfe4610a81ab398c589f594 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/096eb8198cfe4610a81ab398c589f594 2024-11-17T03:14:34,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/096eb8198cfe4610a81ab398c589f594, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:14:34,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/1e9e667b874b4b278fb6259d81ee73a9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1e9e667b874b4b278fb6259d81ee73a9 2024-11-17T03:14:34,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1e9e667b874b4b278fb6259d81ee73a9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:14:34,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for b29e9d27b06f8df48f1da9682bc65b77 in 859ms, sequenceid=15, compaction requested=false 2024-11-17T03:14:34,493 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T03:14:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:34,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T03:14:34,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,539 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T03:14:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:34,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/38092d9db1364b5d885ddfa5ea6d8a9b is 50, key is test_row_0/A:col10/1731813273717/Put/seqid=0 2024-11-17T03:14:34,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741840_1016 (size=12001) 2024-11-17T03:14:34,583 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/38092d9db1364b5d885ddfa5ea6d8a9b 2024-11-17T03:14:34,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bc04b6b0c8794be79e154a839a898ade is 50, key is test_row_0/B:col10/1731813273717/Put/seqid=0 2024-11-17T03:14:34,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741841_1017 (size=12001) 2024-11-17T03:14:34,640 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bc04b6b0c8794be79e154a839a898ade 2024-11-17T03:14:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T03:14:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/6d718a8685814a6eb480b761cbf86d96 is 50, key is test_row_0/C:col10/1731813273717/Put/seqid=0 2024-11-17T03:14:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741842_1018 (size=12001) 2024-11-17T03:14:34,692 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/6d718a8685814a6eb480b761cbf86d96 2024-11-17T03:14:34,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/38092d9db1364b5d885ddfa5ea6d8a9b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38092d9db1364b5d885ddfa5ea6d8a9b 2024-11-17T03:14:34,733 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38092d9db1364b5d885ddfa5ea6d8a9b, entries=150, sequenceid=34, filesize=11.7 K 2024-11-17T03:14:34,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bc04b6b0c8794be79e154a839a898ade as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc04b6b0c8794be79e154a839a898ade 2024-11-17T03:14:34,756 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc04b6b0c8794be79e154a839a898ade, entries=150, sequenceid=34, filesize=11.7 K 2024-11-17T03:14:34,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/6d718a8685814a6eb480b761cbf86d96 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6d718a8685814a6eb480b761cbf86d96 2024-11-17T03:14:34,778 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6d718a8685814a6eb480b761cbf86d96, entries=150, sequenceid=34, filesize=11.7 K 2024-11-17T03:14:34,786 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for b29e9d27b06f8df48f1da9682bc65b77 in 246ms, sequenceid=34, compaction requested=false 2024-11-17T03:14:34,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:34,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:34,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T03:14:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T03:14:34,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T03:14:34,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2560 sec 2024-11-17T03:14:34,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2770 sec 2024-11-17T03:14:34,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:14:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:34,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:34,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:34,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:34,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:34,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:34,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:35,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/48131e4b74384361ab419cd2a81603ae is 50, key is test_row_0/A:col10/1731813274991/Put/seqid=0 2024-11-17T03:14:35,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741843_1019 (size=14341) 2024-11-17T03:14:35,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/48131e4b74384361ab419cd2a81603ae 2024-11-17T03:14:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813335044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b7a6165c02e34ee2bec1cd3a96dc9b3c is 50, key is test_row_0/B:col10/1731813274991/Put/seqid=0 2024-11-17T03:14:35,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813335049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813335052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813335056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813335062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741844_1020 (size=12001) 2024-11-17T03:14:35,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b7a6165c02e34ee2bec1cd3a96dc9b3c 2024-11-17T03:14:35,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e4db4777226545af824bc6a9e01d29f4 is 50, key is test_row_0/C:col10/1731813274991/Put/seqid=0 2024-11-17T03:14:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741845_1021 (size=12001) 2024-11-17T03:14:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813335168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813335168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813335179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813335179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813335180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813335377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813335387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813335387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813335397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813335397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e4db4777226545af824bc6a9e01d29f4 2024-11-17T03:14:35,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/48131e4b74384361ab419cd2a81603ae as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/48131e4b74384361ab419cd2a81603ae 2024-11-17T03:14:35,568 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T03:14:35,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/48131e4b74384361ab419cd2a81603ae, entries=200, sequenceid=47, filesize=14.0 K 2024-11-17T03:14:35,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b7a6165c02e34ee2bec1cd3a96dc9b3c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b7a6165c02e34ee2bec1cd3a96dc9b3c 2024-11-17T03:14:35,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b7a6165c02e34ee2bec1cd3a96dc9b3c, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T03:14:35,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e4db4777226545af824bc6a9e01d29f4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4db4777226545af824bc6a9e01d29f4 2024-11-17T03:14:35,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4db4777226545af824bc6a9e01d29f4, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T03:14:35,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b29e9d27b06f8df48f1da9682bc65b77 in 624ms, sequenceid=47, compaction requested=true 2024-11-17T03:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:35,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:35,629 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:35,629 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:35,633 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:35,635 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:35,635 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:35,636 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/096eb8198cfe4610a81ab398c589f594, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc04b6b0c8794be79e154a839a898ade, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b7a6165c02e34ee2bec1cd3a96dc9b3c] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=35.2 K 2024-11-17T03:14:35,637 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:35,637 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:35,637 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 096eb8198cfe4610a81ab398c589f594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813273598 2024-11-17T03:14:35,637 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:35,638 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f6de317de0db41279fe4a50e7d47a621, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38092d9db1364b5d885ddfa5ea6d8a9b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/48131e4b74384361ab419cd2a81603ae] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=37.4 K 2024-11-17T03:14:35,639 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting bc04b6b0c8794be79e154a839a898ade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731813273717 2024-11-17T03:14:35,639 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6de317de0db41279fe4a50e7d47a621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813273598 2024-11-17T03:14:35,640 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38092d9db1364b5d885ddfa5ea6d8a9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731813273717 2024-11-17T03:14:35,640 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a6165c02e34ee2bec1cd3a96dc9b3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813274942 2024-11-17T03:14:35,642 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48131e4b74384361ab419cd2a81603ae, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813274942 2024-11-17T03:14:35,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T03:14:35,664 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T03:14:35,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T03:14:35,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T03:14:35,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T03:14:35,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T03:14:35,675 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T03:14:35,675 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T03:14:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T03:14:35,678 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:35,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T03:14:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T03:14:35,689 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:35,691 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:35,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:35,697 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:35,698 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/90e611bb295f41d58c2ecac158f6db5d is 50, key is test_row_0/B:col10/1731813274991/Put/seqid=0 2024-11-17T03:14:35,704 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:35,705 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/74898e9355514095ae49046364821894 is 50, key is test_row_0/A:col10/1731813274991/Put/seqid=0 2024-11-17T03:14:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:35,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T03:14:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:35,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741846_1022 (size=12104) 2024-11-17T03:14:35,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d38b7dac745542038cdfc48f0e9a0ddc is 50, key is test_row_0/A:col10/1731813275041/Put/seqid=0 2024-11-17T03:14:35,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741847_1023 (size=12104) 2024-11-17T03:14:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813335731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813335735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,755 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/90e611bb295f41d58c2ecac158f6db5d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/90e611bb295f41d58c2ecac158f6db5d 2024-11-17T03:14:35,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813335747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813335747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813335750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,768 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/74898e9355514095ae49046364821894 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/74898e9355514095ae49046364821894 2024-11-17T03:14:35,780 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into 90e611bb295f41d58c2ecac158f6db5d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:35,780 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:35,783 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=13, startTime=1731813275626; duration=0sec 2024-11-17T03:14:35,784 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:35,784 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:35,784 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T03:14:35,789 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:35,789 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:35,789 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:35,790 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1e9e667b874b4b278fb6259d81ee73a9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6d718a8685814a6eb480b761cbf86d96, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4db4777226545af824bc6a9e01d29f4] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=35.2 K 2024-11-17T03:14:35,791 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e9e667b874b4b278fb6259d81ee73a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813273598 2024-11-17T03:14:35,792 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d718a8685814a6eb480b761cbf86d96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731813273717 2024-11-17T03:14:35,793 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e4db4777226545af824bc6a9e01d29f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813274942 2024-11-17T03:14:35,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741848_1024 (size=14341) 2024-11-17T03:14:35,796 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into 74898e9355514095ae49046364821894(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:35,796 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:35,796 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=13, startTime=1731813275623; duration=0sec 2024-11-17T03:14:35,797 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:35,798 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:35,839 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:35,841 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/72c778c9fba84d75b3d5aa4d0d975666 is 50, key is test_row_0/C:col10/1731813274991/Put/seqid=0 2024-11-17T03:14:35,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:35,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:35,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:35,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:35,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:35,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741849_1025 (size=12104) 2024-11-17T03:14:35,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813335858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813335858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813335868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813335868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813335869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:35,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:35,881 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/72c778c9fba84d75b3d5aa4d0d975666 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/72c778c9fba84d75b3d5aa4d0d975666 2024-11-17T03:14:35,949 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into 72c778c9fba84d75b3d5aa4d0d975666(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:35,949 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:35,949 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=13, startTime=1731813275626; duration=0sec 2024-11-17T03:14:35,949 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:35,949 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T03:14:36,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:36,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:36,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813336067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813336067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813336077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813336088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813336088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d38b7dac745542038cdfc48f0e9a0ddc 2024-11-17T03:14:36,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/da76a10ea4db4fb1a64f14d396e17d21 is 50, key is test_row_0/B:col10/1731813275041/Put/seqid=0 2024-11-17T03:14:36,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741850_1026 (size=12001) 2024-11-17T03:14:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T03:14:36,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:36,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T03:14:36,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813336378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813336378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813336388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813336397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813336397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:36,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:36,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/da76a10ea4db4fb1a64f14d396e17d21 2024-11-17T03:14:36,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ebe040ff05ae4e81a44dfe4030703cad is 50, key is test_row_0/C:col10/1731813275041/Put/seqid=0 2024-11-17T03:14:36,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741851_1027 (size=12001) 2024-11-17T03:14:36,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ebe040ff05ae4e81a44dfe4030703cad 2024-11-17T03:14:36,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d38b7dac745542038cdfc48f0e9a0ddc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d38b7dac745542038cdfc48f0e9a0ddc 2024-11-17T03:14:36,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d38b7dac745542038cdfc48f0e9a0ddc, entries=200, sequenceid=67, filesize=14.0 K 2024-11-17T03:14:36,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/da76a10ea4db4fb1a64f14d396e17d21 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/da76a10ea4db4fb1a64f14d396e17d21 2024-11-17T03:14:36,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/da76a10ea4db4fb1a64f14d396e17d21, entries=150, sequenceid=67, filesize=11.7 K 2024-11-17T03:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ebe040ff05ae4e81a44dfe4030703cad as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ebe040ff05ae4e81a44dfe4030703cad 2024-11-17T03:14:36,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ebe040ff05ae4e81a44dfe4030703cad, entries=150, sequenceid=67, filesize=11.7 K 2024-11-17T03:14:36,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b29e9d27b06f8df48f1da9682bc65b77 in 1073ms, sequenceid=67, compaction requested=false 2024-11-17T03:14:36,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:36,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T03:14:36,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:36,792 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:14:36,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:36,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:36,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:36,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/4c7ead0dbf4e4ad18bb6e6be14bcb313 is 50, key is test_row_0/A:col10/1731813275727/Put/seqid=0 2024-11-17T03:14:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T03:14:36,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741852_1028 (size=12001) 2024-11-17T03:14:36,824 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/4c7ead0dbf4e4ad18bb6e6be14bcb313 2024-11-17T03:14:36,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/6989e046799a4044bedb7fb8d68b603b is 50, key is test_row_0/B:col10/1731813275727/Put/seqid=0 2024-11-17T03:14:36,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741853_1029 (size=12001) 2024-11-17T03:14:36,880 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/6989e046799a4044bedb7fb8d68b603b 2024-11-17T03:14:36,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ff454110596f40fd8bebd486ad3c4b20 is 50, key is test_row_0/C:col10/1731813275727/Put/seqid=0 2024-11-17T03:14:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:36,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:36,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741854_1030 (size=12001) 2024-11-17T03:14:36,935 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ff454110596f40fd8bebd486ad3c4b20 2024-11-17T03:14:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813336939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/4c7ead0dbf4e4ad18bb6e6be14bcb313 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4c7ead0dbf4e4ad18bb6e6be14bcb313 2024-11-17T03:14:36,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813336942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813336944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813336945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813336950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:36,969 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4c7ead0dbf4e4ad18bb6e6be14bcb313, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T03:14:36,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/6989e046799a4044bedb7fb8d68b603b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6989e046799a4044bedb7fb8d68b603b 2024-11-17T03:14:36,989 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6989e046799a4044bedb7fb8d68b603b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T03:14:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ff454110596f40fd8bebd486ad3c4b20 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ff454110596f40fd8bebd486ad3c4b20 2024-11-17T03:14:37,007 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ff454110596f40fd8bebd486ad3c4b20, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T03:14:37,009 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b29e9d27b06f8df48f1da9682bc65b77 in 217ms, sequenceid=82, compaction requested=true 2024-11-17T03:14:37,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:37,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:37,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T03:14:37,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T03:14:37,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T03:14:37,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3210 sec 2024-11-17T03:14:37,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3590 sec 2024-11-17T03:14:37,066 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813276936, max timestamp=1731813277058Num uniques 859; 2024-11-17T03:14:37,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813276936, max timestamp=1731813277058Num uniques 859; 2024-11-17T03:14:37,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:37,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:37,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-17T03:14:37,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:37,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813276936, max timestamp=1731813277058Num uniques 859; 2024-11-17T03:14:37,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:37,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:37,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:37,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:37,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:37,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:37,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/890e3e8387ee4277b6c7658155693a17 is 50, key is test_row_0/A:col10/1731813276936/Put/seqid=0 2024-11-17T03:14:37,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741855_1031 (size=12001) 2024-11-17T03:14:37,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/890e3e8387ee4277b6c7658155693a17 2024-11-17T03:14:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813277121, max timestamp=1731813277134Num uniques 859; 2024-11-17T03:14:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:37,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/7807580cd8a343d3916c08f218755fa0 is 50, key is test_row_0/B:col10/1731813276936/Put/seqid=0 2024-11-17T03:14:37,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813277121, max timestamp=1731813277136Num uniques 859; 2024-11-17T03:14:37,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:37,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813277121, max timestamp=1731813277134Num uniques 859; 2024-11-17T03:14:37,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:37,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741856_1032 (size=12001) 2024-11-17T03:14:37,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731813277179, max timestamp=1731813277200Num uniques 1018; 2024-11-17T03:14:37,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T03:14:37,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731813277179, max timestamp=1731813277200Num uniques 1018; 2024-11-17T03:14:37,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-17T03:14:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731813277179, max timestamp=1731813277200Num uniques 1018; 2024-11-17T03:14:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T03:14:37,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731813277217, max timestamp=1731813277254Num uniques 1127; 2024-11-17T03:14:37,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-17T03:14:37,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731813277217, max timestamp=1731813277254Num uniques 1127; 2024-11-17T03:14:37,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-17T03:14:37,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731813277217, max timestamp=1731813277254Num uniques 1009; 2024-11-17T03:14:37,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-11-17T03:14:37,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731813277293, max timestamp=1731813277316Num uniques 1136; 2024-11-17T03:14:37,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731813277293, max timestamp=1731813277316Num uniques 1136; 2024-11-17T03:14:37,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731813277293, max timestamp=1731813277316Num uniques 1136; 2024-11-17T03:14:37,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-11-17T03:14:37,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-11-17T03:14:37,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=17676, suffix heap size=146112,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=386 2024-11-17T03:14:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813337354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813337356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813337357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813337359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813337369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813337478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813337478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813337479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813337479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813337479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/7807580cd8a343d3916c08f218755fa0 2024-11-17T03:14:37,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/31a460f84a714d12bd1530b0c284a247 is 50, key is test_row_0/C:col10/1731813276936/Put/seqid=0 2024-11-17T03:14:37,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741857_1033 (size=12001) 2024-11-17T03:14:37,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/31a460f84a714d12bd1530b0c284a247 2024-11-17T03:14:37,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/890e3e8387ee4277b6c7658155693a17 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/890e3e8387ee4277b6c7658155693a17 2024-11-17T03:14:37,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/890e3e8387ee4277b6c7658155693a17, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T03:14:37,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/7807580cd8a343d3916c08f218755fa0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/7807580cd8a343d3916c08f218755fa0 2024-11-17T03:14:37,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/7807580cd8a343d3916c08f218755fa0, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T03:14:37,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/31a460f84a714d12bd1530b0c284a247 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/31a460f84a714d12bd1530b0c284a247 2024-11-17T03:14:37,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/31a460f84a714d12bd1530b0c284a247, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T03:14:37,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for b29e9d27b06f8df48f1da9682bc65b77 in 607ms, sequenceid=104, compaction requested=true 2024-11-17T03:14:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:37,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:37,679 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:37,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:37,679 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:37,682 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:37,682 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:37,682 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:37,682 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:37,682 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:37,682 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:37,683 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/74898e9355514095ae49046364821894, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d38b7dac745542038cdfc48f0e9a0ddc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4c7ead0dbf4e4ad18bb6e6be14bcb313, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/890e3e8387ee4277b6c7658155693a17] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=49.3 K 2024-11-17T03:14:37,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:37,683 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/90e611bb295f41d58c2ecac158f6db5d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/da76a10ea4db4fb1a64f14d396e17d21, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6989e046799a4044bedb7fb8d68b603b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/7807580cd8a343d3916c08f218755fa0] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=47.0 K 2024-11-17T03:14:37,684 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74898e9355514095ae49046364821894, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813274942 2024-11-17T03:14:37,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:37,684 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e611bb295f41d58c2ecac158f6db5d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813274942 2024-11-17T03:14:37,684 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d38b7dac745542038cdfc48f0e9a0ddc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731813275041 2024-11-17T03:14:37,685 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting da76a10ea4db4fb1a64f14d396e17d21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731813275041 2024-11-17T03:14:37,685 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7ead0dbf4e4ad18bb6e6be14bcb313, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731813275727 2024-11-17T03:14:37,685 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6989e046799a4044bedb7fb8d68b603b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731813275727 2024-11-17T03:14:37,686 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890e3e8387ee4277b6c7658155693a17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731813276936 2024-11-17T03:14:37,686 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7807580cd8a343d3916c08f218755fa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731813276936 2024-11-17T03:14:37,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:37,707 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:37,707 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3bf311228da44419d58cd3a54607624 is 50, key is test_row_0/B:col10/1731813276936/Put/seqid=0 2024-11-17T03:14:37,710 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:37,711 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/494dcc6c31e74ca1b64b08d1dfae869c is 50, key is test_row_0/A:col10/1731813276936/Put/seqid=0 2024-11-17T03:14:37,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741858_1034 (size=12241) 2024-11-17T03:14:37,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741859_1035 (size=12241) 2024-11-17T03:14:37,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=181.14 KB heapSize=503.86 KB 2024-11-17T03:14:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:37,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8910, totalHeapSize=20520, min timestamp=1731813277351, max timestamp=1731813277696Num uniques 1245; 2024-11-17T03:14:37,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57000, new segment data size=8910, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-17T03:14:37,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8914, totalHeapSize=20520, min timestamp=1731813277351, max timestamp=1731813277696Num uniques 1245; 2024-11-17T03:14:37,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=57004, new segment data size=8914, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-17T03:14:37,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8913, totalHeapSize=20520, min timestamp=1731813277351, max timestamp=1731813277696Num uniques 1245; 2024-11-17T03:14:37,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=57003, new segment data size=8913, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-17T03:14:37,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:37,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:37,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:37,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:37,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:37,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:37,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/67f979c9ed274d74b402cd1de57c736a is 50, key is test_row_0/A:col10/1731813277732/Put/seqid=0 2024-11-17T03:14:37,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741860_1036 (size=14541) 2024-11-17T03:14:37,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/67f979c9ed274d74b402cd1de57c736a 2024-11-17T03:14:37,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/d8132433858f4470801adc6c94f29010 is 50, key is test_row_0/B:col10/1731813277732/Put/seqid=0 2024-11-17T03:14:37,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741861_1037 (size=12151) 2024-11-17T03:14:37,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813277774, max timestamp=1731813277801Num uniques 859; 2024-11-17T03:14:37,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813277774, max timestamp=1731813277801Num uniques 859; 2024-11-17T03:14:37,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:14:37,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:14:37,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731813277788, max timestamp=1731813277801Num uniques 859; 2024-11-17T03:14:37,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-17T03:14:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T03:14:37,827 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:37,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T03:14:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T03:14:37,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:37,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:37,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:37,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813337830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813337832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813337834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813337836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813337837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T03:14:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813337947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813337947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813337962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813337962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813337962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:37,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T03:14:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:37,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,128 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3bf311228da44419d58cd3a54607624 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3bf311228da44419d58cd3a54607624 2024-11-17T03:14:38,139 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/494dcc6c31e74ca1b64b08d1dfae869c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/494dcc6c31e74ca1b64b08d1dfae869c 2024-11-17T03:14:38,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T03:14:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T03:14:38,153 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into b3bf311228da44419d58cd3a54607624(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:38,153 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into 494dcc6c31e74ca1b64b08d1dfae869c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:38,153 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:38,153 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:38,153 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=12, startTime=1731813277678; duration=0sec 2024-11-17T03:14:38,153 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=12, startTime=1731813277679; duration=0sec 2024-11-17T03:14:38,154 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:38,154 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:38,154 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:38,155 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:38,155 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:38,158 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:38,158 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:38,158 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,158 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/72c778c9fba84d75b3d5aa4d0d975666, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ebe040ff05ae4e81a44dfe4030703cad, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ff454110596f40fd8bebd486ad3c4b20, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/31a460f84a714d12bd1530b0c284a247] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=47.0 K 2024-11-17T03:14:38,159 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c778c9fba84d75b3d5aa4d0d975666, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813274942 2024-11-17T03:14:38,160 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe040ff05ae4e81a44dfe4030703cad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731813275041 2024-11-17T03:14:38,161 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ff454110596f40fd8bebd486ad3c4b20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731813275727 2024-11-17T03:14:38,161 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a460f84a714d12bd1530b0c284a247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731813276936 2024-11-17T03:14:38,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813338157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813338167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813338157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813338177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813338178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/d8132433858f4470801adc6c94f29010 2024-11-17T03:14:38,219 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:38,220 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e0cb3416c5af4bc8b6b035bcc9c7f093 is 50, key is test_row_0/C:col10/1731813276936/Put/seqid=0 2024-11-17T03:14:38,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ddb112005c904ca59ecdcb51be498831 is 50, key is test_row_0/C:col10/1731813277732/Put/seqid=0 2024-11-17T03:14:38,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741862_1038 (size=12241) 2024-11-17T03:14:38,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741863_1039 (size=12151) 2024-11-17T03:14:38,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T03:14:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T03:14:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T03:14:38,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813338478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813338478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813338487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813338487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813338496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:38,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T03:14:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:38,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ddb112005c904ca59ecdcb51be498831 2024-11-17T03:14:38,680 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e0cb3416c5af4bc8b6b035bcc9c7f093 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e0cb3416c5af4bc8b6b035bcc9c7f093 2024-11-17T03:14:38,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/67f979c9ed274d74b402cd1de57c736a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/67f979c9ed274d74b402cd1de57c736a 2024-11-17T03:14:38,693 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into e0cb3416c5af4bc8b6b035bcc9c7f093(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:38,693 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:38,693 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=12, startTime=1731813277683; duration=0sec 2024-11-17T03:14:38,693 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:38,693 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:38,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/67f979c9ed274d74b402cd1de57c736a, entries=200, sequenceid=196, filesize=14.2 K 2024-11-17T03:14:38,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/d8132433858f4470801adc6c94f29010 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/d8132433858f4470801adc6c94f29010 2024-11-17T03:14:38,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/d8132433858f4470801adc6c94f29010, entries=150, sequenceid=196, filesize=11.9 K 2024-11-17T03:14:38,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ddb112005c904ca59ecdcb51be498831 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ddb112005c904ca59ecdcb51be498831 2024-11-17T03:14:38,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ddb112005c904ca59ecdcb51be498831, entries=150, sequenceid=196, filesize=11.9 K 2024-11-17T03:14:38,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~122.58 KB/125520, currentSize=140.49 KB/143865 for b29e9d27b06f8df48f1da9682bc65b77 in 992ms, sequenceid=196, compaction requested=false 2024-11-17T03:14:38,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:38,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T03:14:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,763 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=140.49 KB heapSize=394.62 KB 2024-11-17T03:14:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:38,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a8f3c19ccd49407a999a56ac9802e18e is 50, key is test_row_0/A:col10/1731813277834/Put/seqid=0 2024-11-17T03:14:38,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741864_1040 (size=12151) 2024-11-17T03:14:38,805 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a8f3c19ccd49407a999a56ac9802e18e 2024-11-17T03:14:38,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/91adce760a484c6482ad92bebefd9021 is 50, key is test_row_0/B:col10/1731813277834/Put/seqid=0 2024-11-17T03:14:38,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741865_1041 (size=12151) 2024-11-17T03:14:38,850 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/91adce760a484c6482ad92bebefd9021 2024-11-17T03:14:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ac54f270a4ec41f1bb5e0de114407e05 is 50, key is test_row_0/C:col10/1731813277834/Put/seqid=0 2024-11-17T03:14:38,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741866_1042 (size=12151) 2024-11-17T03:14:38,907 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ac54f270a4ec41f1bb5e0de114407e05 2024-11-17T03:14:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a8f3c19ccd49407a999a56ac9802e18e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a8f3c19ccd49407a999a56ac9802e18e 2024-11-17T03:14:38,931 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a8f3c19ccd49407a999a56ac9802e18e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T03:14:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/91adce760a484c6482ad92bebefd9021 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/91adce760a484c6482ad92bebefd9021 2024-11-17T03:14:38,947 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/91adce760a484c6482ad92bebefd9021, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T03:14:38,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ac54f270a4ec41f1bb5e0de114407e05 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ac54f270a4ec41f1bb5e0de114407e05 2024-11-17T03:14:38,962 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ac54f270a4ec41f1bb5e0de114407e05, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T03:14:38,965 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~394.57 KB/404040, currentSize=0 B/0 for b29e9d27b06f8df48f1da9682bc65b77 in 202ms, sequenceid=235, compaction requested=true 2024-11-17T03:14:38,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:38,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:38,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T03:14:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T03:14:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T03:14:38,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T03:14:38,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-11-17T03:14:38,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1490 sec 2024-11-17T03:14:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:39,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:14:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:39,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:39,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:39,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:39,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:39,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/9071396afcc04ab9bd35d2f34f436fe3 is 50, key is test_row_0/A:col10/1731813279008/Put/seqid=0 2024-11-17T03:14:39,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741867_1043 (size=12151) 2024-11-17T03:14:39,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/9071396afcc04ab9bd35d2f34f436fe3 2024-11-17T03:14:39,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813279051, max timestamp=1731813279071Num uniques 859; 2024-11-17T03:14:39,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813279051, max timestamp=1731813279071Num uniques 859; 2024-11-17T03:14:39,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813279051, max timestamp=1731813279071Num uniques 859; 2024-11-17T03:14:39,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:39,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:39,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:39,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f is 50, key is test_row_0/B:col10/1731813279008/Put/seqid=0 2024-11-17T03:14:39,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741868_1044 (size=12151) 2024-11-17T03:14:39,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f 2024-11-17T03:14:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813339112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/437f5b7c20054ba0a7b52fdf937043aa is 50, key is test_row_0/C:col10/1731813279008/Put/seqid=0 2024-11-17T03:14:39,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813339115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813339116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813339120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813339124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741869_1045 (size=12151) 2024-11-17T03:14:39,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/437f5b7c20054ba0a7b52fdf937043aa 2024-11-17T03:14:39,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/9071396afcc04ab9bd35d2f34f436fe3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/9071396afcc04ab9bd35d2f34f436fe3 2024-11-17T03:14:39,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/9071396afcc04ab9bd35d2f34f436fe3, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T03:14:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f 2024-11-17T03:14:39,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T03:14:39,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/437f5b7c20054ba0a7b52fdf937043aa as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/437f5b7c20054ba0a7b52fdf937043aa 2024-11-17T03:14:39,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/437f5b7c20054ba0a7b52fdf937043aa, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T03:14:39,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b29e9d27b06f8df48f1da9682bc65b77 in 193ms, sequenceid=245, compaction requested=true 2024-11-17T03:14:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:39,207 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:39,207 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:39,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:39,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:39,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:39,211 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:39,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:39,211 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:39,211 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:39,211 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:39,211 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:39,211 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:39,211 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/494dcc6c31e74ca1b64b08d1dfae869c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/67f979c9ed274d74b402cd1de57c736a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a8f3c19ccd49407a999a56ac9802e18e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/9071396afcc04ab9bd35d2f34f436fe3] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=49.9 K 2024-11-17T03:14:39,211 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3bf311228da44419d58cd3a54607624, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/d8132433858f4470801adc6c94f29010, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/91adce760a484c6482ad92bebefd9021, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=47.6 K 2024-11-17T03:14:39,212 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 494dcc6c31e74ca1b64b08d1dfae869c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731813276936 2024-11-17T03:14:39,212 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b3bf311228da44419d58cd3a54607624, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731813276936 2024-11-17T03:14:39,213 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f979c9ed274d74b402cd1de57c736a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731813277365 2024-11-17T03:14:39,214 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting d8132433858f4470801adc6c94f29010, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731813277365 2024-11-17T03:14:39,214 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8f3c19ccd49407a999a56ac9802e18e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731813277824 2024-11-17T03:14:39,215 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9071396afcc04ab9bd35d2f34f436fe3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731813279004 2024-11-17T03:14:39,215 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 91adce760a484c6482ad92bebefd9021, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731813277824 2024-11-17T03:14:39,216 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting fbfaa6ffbdcb4f0bbb4a3e39ccf5363f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731813279004 2024-11-17T03:14:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:39,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T03:14:39,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:39,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:39,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:39,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:39,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:39,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:39,256 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:39,257 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/f36d5d399ef041dd9bae4d2c6b04999f is 50, key is test_row_0/A:col10/1731813279008/Put/seqid=0 2024-11-17T03:14:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,262 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#34 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813339254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,263 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/6e57b6e847874055a90a764ba778da82 is 50, key is test_row_0/B:col10/1731813279008/Put/seqid=0 2024-11-17T03:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/6f9d6f1f50c546dcb80fcf8178a29850 is 50, key is test_row_0/A:col10/1731813279120/Put/seqid=0 2024-11-17T03:14:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813339249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813339261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813339263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813339263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741870_1046 (size=12527) 2024-11-17T03:14:39,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741871_1047 (size=14741) 2024-11-17T03:14:39,335 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/f36d5d399ef041dd9bae4d2c6b04999f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f36d5d399ef041dd9bae4d2c6b04999f 2024-11-17T03:14:39,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/6f9d6f1f50c546dcb80fcf8178a29850 2024-11-17T03:14:39,349 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into f36d5d399ef041dd9bae4d2c6b04999f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:39,349 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:39,349 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=12, startTime=1731813279207; duration=0sec 2024-11-17T03:14:39,349 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:39,349 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:39,349 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:39,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741872_1048 (size=12527) 2024-11-17T03:14:39,358 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:39,359 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:39,359 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:39,359 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e0cb3416c5af4bc8b6b035bcc9c7f093, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ddb112005c904ca59ecdcb51be498831, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ac54f270a4ec41f1bb5e0de114407e05, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/437f5b7c20054ba0a7b52fdf937043aa] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=47.6 K 2024-11-17T03:14:39,362 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0cb3416c5af4bc8b6b035bcc9c7f093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731813276936 2024-11-17T03:14:39,363 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb112005c904ca59ecdcb51be498831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731813277365 2024-11-17T03:14:39,363 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac54f270a4ec41f1bb5e0de114407e05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731813277824 2024-11-17T03:14:39,364 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437f5b7c20054ba0a7b52fdf937043aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731813279004 2024-11-17T03:14:39,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a5ce08fdebe44b57b6c737a6211c4b76 is 50, key is test_row_0/B:col10/1731813279120/Put/seqid=0 2024-11-17T03:14:39,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813339367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741873_1049 (size=12301) 2024-11-17T03:14:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813339386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a5ce08fdebe44b57b6c737a6211c4b76 2024-11-17T03:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813339386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813339386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813339387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,397 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#37 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:39,398 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/084103abfde045b5a75eee08c4ee86a5 is 50, key is test_row_0/C:col10/1731813279008/Put/seqid=0 2024-11-17T03:14:39,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741874_1050 (size=12527) 2024-11-17T03:14:39,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/deb56902a557447d9d05def0dae1ba32 is 50, key is test_row_0/C:col10/1731813279120/Put/seqid=0 2024-11-17T03:14:39,444 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/084103abfde045b5a75eee08c4ee86a5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/084103abfde045b5a75eee08c4ee86a5 2024-11-17T03:14:39,459 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into 084103abfde045b5a75eee08c4ee86a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:39,459 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:39,460 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=12, startTime=1731813279209; duration=0sec 2024-11-17T03:14:39,460 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:39,460 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:39,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741875_1051 (size=12301) 2024-11-17T03:14:39,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/deb56902a557447d9d05def0dae1ba32 2024-11-17T03:14:39,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/6f9d6f1f50c546dcb80fcf8178a29850 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/6f9d6f1f50c546dcb80fcf8178a29850 2024-11-17T03:14:39,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/6f9d6f1f50c546dcb80fcf8178a29850, entries=200, sequenceid=282, filesize=14.4 K 2024-11-17T03:14:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a5ce08fdebe44b57b6c737a6211c4b76 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a5ce08fdebe44b57b6c737a6211c4b76 2024-11-17T03:14:39,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a5ce08fdebe44b57b6c737a6211c4b76, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T03:14:39,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/deb56902a557447d9d05def0dae1ba32 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/deb56902a557447d9d05def0dae1ba32 2024-11-17T03:14:39,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/deb56902a557447d9d05def0dae1ba32, entries=150, sequenceid=282, filesize=12.0 K 2024-11-17T03:14:39,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for b29e9d27b06f8df48f1da9682bc65b77 in 286ms, sequenceid=282, compaction requested=false 2024-11-17T03:14:39,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:39,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:14:39,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:39,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:39,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:39,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:39,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:39,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:39,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/83de486b352d4395a4e4bea743521e4f is 50, key is test_row_0/A:col10/1731813279260/Put/seqid=0 2024-11-17T03:14:39,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741876_1052 (size=14741) 2024-11-17T03:14:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813339624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813279605, max timestamp=1731813279629Num uniques 859; 2024-11-17T03:14:39,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813279605, max timestamp=1731813279629Num uniques 859; 2024-11-17T03:14:39,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813279605, max timestamp=1731813279629Num uniques 859; 2024-11-17T03:14:39,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:39,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:39,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:14:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813339663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813339669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813339672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813339675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813339737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,764 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/6e57b6e847874055a90a764ba778da82 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6e57b6e847874055a90a764ba778da82 2024-11-17T03:14:39,776 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into 6e57b6e847874055a90a764ba778da82(size=12.2 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:39,776 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:39,778 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=12, startTime=1731813279207; duration=0sec 2024-11-17T03:14:39,778 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:39,779 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813339778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813339778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813339787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813339788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813339947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T03:14:39,980 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:39,982 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T03:14:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T03:14:39,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:39,990 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:39,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813339997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813339997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813340007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813340007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/83de486b352d4395a4e4bea743521e4f 2024-11-17T03:14:40,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/697397bd974b4140b53418e666a08906 is 50, key is test_row_0/B:col10/1731813279260/Put/seqid=0 2024-11-17T03:14:40,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741877_1053 (size=12301) 2024-11-17T03:14:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T03:14:40,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:40,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:40,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813340267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T03:14:40,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813340307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813340307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813340318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813340318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/697397bd974b4140b53418e666a08906 2024-11-17T03:14:40,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/3c9a59823de148eb9957b5f5ff4a22e3 is 50, key is test_row_0/C:col10/1731813279260/Put/seqid=0 2024-11-17T03:14:40,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741878_1054 (size=12301) 2024-11-17T03:14:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T03:14:40,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813340787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813340817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813340817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813340837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813340837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:40,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/3c9a59823de148eb9957b5f5ff4a22e3 2024-11-17T03:14:40,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:40,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:40,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/83de486b352d4395a4e4bea743521e4f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/83de486b352d4395a4e4bea743521e4f 2024-11-17T03:14:40,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/83de486b352d4395a4e4bea743521e4f, entries=200, sequenceid=295, filesize=14.4 K 2024-11-17T03:14:41,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/697397bd974b4140b53418e666a08906 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/697397bd974b4140b53418e666a08906 2024-11-17T03:14:41,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/697397bd974b4140b53418e666a08906, entries=150, sequenceid=295, filesize=12.0 K 2024-11-17T03:14:41,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/3c9a59823de148eb9957b5f5ff4a22e3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/3c9a59823de148eb9957b5f5ff4a22e3 2024-11-17T03:14:41,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/3c9a59823de148eb9957b5f5ff4a22e3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-17T03:14:41,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for b29e9d27b06f8df48f1da9682bc65b77 in 1463ms, sequenceid=295, compaction requested=true 2024-11-17T03:14:41,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:41,046 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:41,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:41,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:41,049 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:41,051 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:41,051 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:41,051 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:41,051 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:41,051 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:41,051 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:41,052 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6e57b6e847874055a90a764ba778da82, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a5ce08fdebe44b57b6c737a6211c4b76, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/697397bd974b4140b53418e666a08906] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.3 K 2024-11-17T03:14:41,052 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f36d5d399ef041dd9bae4d2c6b04999f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/6f9d6f1f50c546dcb80fcf8178a29850, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/83de486b352d4395a4e4bea743521e4f] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=41.0 K 2024-11-17T03:14:41,052 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e57b6e847874055a90a764ba778da82, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731813279004 2024-11-17T03:14:41,053 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ce08fdebe44b57b6c737a6211c4b76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731813279081 2024-11-17T03:14:41,055 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36d5d399ef041dd9bae4d2c6b04999f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731813279004 2024-11-17T03:14:41,058 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9d6f1f50c546dcb80fcf8178a29850, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731813279081 2024-11-17T03:14:41,059 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83de486b352d4395a4e4bea743521e4f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731813279246 2024-11-17T03:14:41,062 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 697397bd974b4140b53418e666a08906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731813279246 2024-11-17T03:14:41,089 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:41,090 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a409dfcc3f8342c6890952e8a5fa9d38 is 50, key is test_row_0/B:col10/1731813279260/Put/seqid=0 2024-11-17T03:14:41,096 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:41,097 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a9f8b8c77391403cb1f3ce5213934df1 is 50, key is test_row_0/A:col10/1731813279260/Put/seqid=0 2024-11-17T03:14:41,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T03:14:41,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:41,122 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-17T03:14:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:41,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741879_1055 (size=12779) 2024-11-17T03:14:41,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T03:14:41,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741880_1056 (size=12779) 2024-11-17T03:14:41,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/7d24f3f01a1142c5a0ef6110e9e03ce4 is 50, key is test_row_0/A:col10/1731813279661/Put/seqid=0 2024-11-17T03:14:41,148 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a9f8b8c77391403cb1f3ce5213934df1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a9f8b8c77391403cb1f3ce5213934df1 2024-11-17T03:14:41,164 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into a9f8b8c77391403cb1f3ce5213934df1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:41,164 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:41,165 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=13, startTime=1731813281046; duration=0sec 2024-11-17T03:14:41,165 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:41,165 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:41,165 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:41,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741881_1057 (size=12301) 2024-11-17T03:14:41,168 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:41,168 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:41,168 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:41,168 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/084103abfde045b5a75eee08c4ee86a5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/deb56902a557447d9d05def0dae1ba32, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/3c9a59823de148eb9957b5f5ff4a22e3] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.3 K 2024-11-17T03:14:41,169 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084103abfde045b5a75eee08c4ee86a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731813279004 2024-11-17T03:14:41,169 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting deb56902a557447d9d05def0dae1ba32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731813279081 2024-11-17T03:14:41,170 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c9a59823de148eb9957b5f5ff4a22e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731813279246 2024-11-17T03:14:41,183 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:41,185 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e20847b1a477465a8a58b64464c00622 is 50, key is test_row_0/C:col10/1731813279260/Put/seqid=0 2024-11-17T03:14:41,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741882_1058 (size=12779) 2024-11-17T03:14:41,530 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a409dfcc3f8342c6890952e8a5fa9d38 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a409dfcc3f8342c6890952e8a5fa9d38 2024-11-17T03:14:41,537 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into a409dfcc3f8342c6890952e8a5fa9d38(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:41,537 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:41,537 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=13, startTime=1731813281048; duration=0sec 2024-11-17T03:14:41,538 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:41,538 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:41,569 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/7d24f3f01a1142c5a0ef6110e9e03ce4 2024-11-17T03:14:41,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bc098513ec4c44089eb234786b86c18b is 50, key is test_row_0/B:col10/1731813279661/Put/seqid=0 2024-11-17T03:14:41,606 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e20847b1a477465a8a58b64464c00622 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e20847b1a477465a8a58b64464c00622 2024-11-17T03:14:41,614 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into e20847b1a477465a8a58b64464c00622(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:41,615 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:41,615 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=13, startTime=1731813281049; duration=0sec 2024-11-17T03:14:41,615 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:41,615 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:41,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741883_1059 (size=12301) 2024-11-17T03:14:41,626 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bc098513ec4c44089eb234786b86c18b 2024-11-17T03:14:41,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/8482a62d2c8f425a9e6381bf8ee8aaf0 is 50, key is test_row_0/C:col10/1731813279661/Put/seqid=0 2024-11-17T03:14:41,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741884_1060 (size=12301) 2024-11-17T03:14:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:41,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813341857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813341866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813341866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813341866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813341867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,873 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,875 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813341977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813341977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813341977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,059 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/8482a62d2c8f425a9e6381bf8ee8aaf0 2024-11-17T03:14:42,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/7d24f3f01a1142c5a0ef6110e9e03ce4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7d24f3f01a1142c5a0ef6110e9e03ce4 2024-11-17T03:14:42,073 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7d24f3f01a1142c5a0ef6110e9e03ce4, entries=150, sequenceid=331, filesize=12.0 K 2024-11-17T03:14:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bc098513ec4c44089eb234786b86c18b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc098513ec4c44089eb234786b86c18b 2024-11-17T03:14:42,089 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc098513ec4c44089eb234786b86c18b, entries=150, sequenceid=331, filesize=12.0 K 2024-11-17T03:14:42,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/8482a62d2c8f425a9e6381bf8ee8aaf0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/8482a62d2c8f425a9e6381bf8ee8aaf0 2024-11-17T03:14:42,102 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/8482a62d2c8f425a9e6381bf8ee8aaf0, entries=150, sequenceid=331, filesize=12.0 K 2024-11-17T03:14:42,107 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=67.09 KB/68700 for b29e9d27b06f8df48f1da9682bc65b77 in 986ms, sequenceid=331, compaction requested=false 2024-11-17T03:14:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T03:14:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T03:14:42,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T03:14:42,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-17T03:14:42,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1330 sec 2024-11-17T03:14:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T03:14:42,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:42,139 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T03:14:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T03:14:42,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:42,143 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:42,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:42,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:14:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:42,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:42,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/b72572d430584a9ebea89161a51f7817 is 50, key is test_row_0/A:col10/1731813281844/Put/seqid=0 2024-11-17T03:14:42,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741885_1061 (size=12301) 2024-11-17T03:14:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813342231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813342232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813342234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T03:14:42,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813342347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813342347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813342348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:42,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:42,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T03:14:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813342557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813342557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813342557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/b72572d430584a9ebea89161a51f7817 2024-11-17T03:14:42,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/cc4aed477ea945f4af9fe25f047cada4 is 50, key is test_row_0/B:col10/1731813281844/Put/seqid=0 2024-11-17T03:14:42,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741886_1062 (size=12301) 2024-11-17T03:14:42,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T03:14:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813342867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813342867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813342867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:42,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:42,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:42,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:42,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/cc4aed477ea945f4af9fe25f047cada4 2024-11-17T03:14:43,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/a417d764e4a244238525babbfb8eb8f1 is 50, key is test_row_0/C:col10/1731813281844/Put/seqid=0 2024-11-17T03:14:43,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741887_1063 (size=12301) 2024-11-17T03:14:43,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T03:14:43,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813343377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813343377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:43,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813343377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:43,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/a417d764e4a244238525babbfb8eb8f1 2024-11-17T03:14:43,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/b72572d430584a9ebea89161a51f7817 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b72572d430584a9ebea89161a51f7817 2024-11-17T03:14:43,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b72572d430584a9ebea89161a51f7817, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T03:14:43,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/cc4aed477ea945f4af9fe25f047cada4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cc4aed477ea945f4af9fe25f047cada4 2024-11-17T03:14:43,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cc4aed477ea945f4af9fe25f047cada4, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T03:14:43,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/a417d764e4a244238525babbfb8eb8f1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a417d764e4a244238525babbfb8eb8f1 2024-11-17T03:14:43,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a417d764e4a244238525babbfb8eb8f1, entries=150, sequenceid=349, filesize=12.0 K 2024-11-17T03:14:43,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b29e9d27b06f8df48f1da9682bc65b77 in 1291ms, sequenceid=349, compaction requested=true 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:43,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:43,484 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:43,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:43,490 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:43,490 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:43,490 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,490 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a9f8b8c77391403cb1f3ce5213934df1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7d24f3f01a1142c5a0ef6110e9e03ce4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b72572d430584a9ebea89161a51f7817] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.5 K 2024-11-17T03:14:43,490 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:43,491 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:43,491 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,491 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f8b8c77391403cb1f3ce5213934df1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731813279246 2024-11-17T03:14:43,491 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a409dfcc3f8342c6890952e8a5fa9d38, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc098513ec4c44089eb234786b86c18b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cc4aed477ea945f4af9fe25f047cada4] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.5 K 2024-11-17T03:14:43,491 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d24f3f01a1142c5a0ef6110e9e03ce4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731813279661 2024-11-17T03:14:43,491 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting a409dfcc3f8342c6890952e8a5fa9d38, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731813279246 2024-11-17T03:14:43,492 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting bc098513ec4c44089eb234786b86c18b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731813279661 2024-11-17T03:14:43,492 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72572d430584a9ebea89161a51f7817, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731813281844 2024-11-17T03:14:43,492 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting cc4aed477ea945f4af9fe25f047cada4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731813281844 2024-11-17T03:14:43,503 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:43,503 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:43,503 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/dce1e2701ebf469899116b6c1efd0e9c is 50, key is test_row_0/B:col10/1731813281844/Put/seqid=0 2024-11-17T03:14:43,504 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/7241231d7e28454f91e93d778be67f79 is 50, key is test_row_0/A:col10/1731813281844/Put/seqid=0 2024-11-17T03:14:43,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741888_1064 (size=12881) 2024-11-17T03:14:43,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741889_1065 (size=12881) 2024-11-17T03:14:43,515 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/7241231d7e28454f91e93d778be67f79 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7241231d7e28454f91e93d778be67f79 2024-11-17T03:14:43,522 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into 7241231d7e28454f91e93d778be67f79(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:43,522 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:43,522 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=13, startTime=1731813283484; duration=0sec 2024-11-17T03:14:43,522 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:43,522 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:43,522 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:43,524 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:43,524 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:43,524 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,524 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e20847b1a477465a8a58b64464c00622, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/8482a62d2c8f425a9e6381bf8ee8aaf0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a417d764e4a244238525babbfb8eb8f1] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.5 K 2024-11-17T03:14:43,524 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20847b1a477465a8a58b64464c00622, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731813279246 2024-11-17T03:14:43,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T03:14:43,525 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8482a62d2c8f425a9e6381bf8ee8aaf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731813279661 2024-11-17T03:14:43,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,526 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:14:43,526 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a417d764e4a244238525babbfb8eb8f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731813281844 2024-11-17T03:14:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/69ded74da5d945ed9efdbcbd72a0418c is 50, key is test_row_0/A:col10/1731813282232/Put/seqid=0 2024-11-17T03:14:43,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741890_1066 (size=12301) 2024-11-17T03:14:43,538 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/69ded74da5d945ed9efdbcbd72a0418c 2024-11-17T03:14:43,546 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#54 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:43,547 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/381179b01b7742fd95d8c1e228a5081c is 50, key is test_row_0/C:col10/1731813281844/Put/seqid=0 2024-11-17T03:14:43,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/ed74b22f8fc545a6a676892834827be9 is 50, key is test_row_0/B:col10/1731813282232/Put/seqid=0 2024-11-17T03:14:43,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741891_1067 (size=12881) 2024-11-17T03:14:43,587 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/381179b01b7742fd95d8c1e228a5081c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/381179b01b7742fd95d8c1e228a5081c 2024-11-17T03:14:43,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741892_1068 (size=12301) 2024-11-17T03:14:43,590 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/ed74b22f8fc545a6a676892834827be9 2024-11-17T03:14:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ce6a7c7fdd24430898a3048e1252b045 is 50, key is test_row_0/C:col10/1731813282232/Put/seqid=0 2024-11-17T03:14:43,606 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into 381179b01b7742fd95d8c1e228a5081c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:43,606 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:43,607 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=13, startTime=1731813283484; duration=0sec 2024-11-17T03:14:43,607 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:43,607 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:43,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741893_1069 (size=12301) 2024-11-17T03:14:43,619 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ce6a7c7fdd24430898a3048e1252b045 2024-11-17T03:14:43,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/69ded74da5d945ed9efdbcbd72a0418c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/69ded74da5d945ed9efdbcbd72a0418c 2024-11-17T03:14:43,637 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/69ded74da5d945ed9efdbcbd72a0418c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T03:14:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/ed74b22f8fc545a6a676892834827be9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ed74b22f8fc545a6a676892834827be9 2024-11-17T03:14:43,657 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ed74b22f8fc545a6a676892834827be9, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T03:14:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ce6a7c7fdd24430898a3048e1252b045 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ce6a7c7fdd24430898a3048e1252b045 2024-11-17T03:14:43,670 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ce6a7c7fdd24430898a3048e1252b045, entries=150, sequenceid=367, filesize=12.0 K 2024-11-17T03:14:43,671 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for b29e9d27b06f8df48f1da9682bc65b77 in 146ms, sequenceid=367, compaction requested=false 2024-11-17T03:14:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:43,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T03:14:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T03:14:43,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T03:14:43,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5300 sec 2024-11-17T03:14:43,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5360 sec 2024-11-17T03:14:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:43,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:14:43,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:43,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:43,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:43,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:43,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:43,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:43,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d44c2d9610f043afb756e947fe040149 is 50, key is test_row_0/A:col10/1731813283893/Put/seqid=0 2024-11-17T03:14:43,917 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/dce1e2701ebf469899116b6c1efd0e9c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/dce1e2701ebf469899116b6c1efd0e9c 2024-11-17T03:14:43,925 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into dce1e2701ebf469899116b6c1efd0e9c(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:43,925 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:43,925 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=13, startTime=1731813283484; duration=0sec 2024-11-17T03:14:43,925 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:43,925 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:43,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741894_1070 (size=17181) 2024-11-17T03:14:43,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d44c2d9610f043afb756e947fe040149 2024-11-17T03:14:43,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/01697bf16b3640419586e24856c36af3 is 50, key is test_row_0/B:col10/1731813283893/Put/seqid=0 2024-11-17T03:14:43,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813283969, max timestamp=1731813283990Num uniques 859; 2024-11-17T03:14:43,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741895_1071 (size=12301) 2024-11-17T03:14:43,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813283969, max timestamp=1731813283988Num uniques 859; 2024-11-17T03:14:43,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813283969, max timestamp=1731813283990Num uniques 859; 2024-11-17T03:14:43,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:43,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:43,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:44,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/01697bf16b3640419586e24856c36af3 2024-11-17T03:14:44,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/a4d996227e884d0897e1ebe16b36e1f0 is 50, key is test_row_0/C:col10/1731813283893/Put/seqid=0 2024-11-17T03:14:44,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741896_1072 (size=12301) 2024-11-17T03:14:44,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/a4d996227e884d0897e1ebe16b36e1f0 2024-11-17T03:14:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813344074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813344076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d44c2d9610f043afb756e947fe040149 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d44c2d9610f043afb756e947fe040149 2024-11-17T03:14:44,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d44c2d9610f043afb756e947fe040149, entries=250, sequenceid=378, filesize=16.8 K 2024-11-17T03:14:44,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/01697bf16b3640419586e24856c36af3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/01697bf16b3640419586e24856c36af3 2024-11-17T03:14:44,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/01697bf16b3640419586e24856c36af3, entries=150, sequenceid=378, filesize=12.0 K 2024-11-17T03:14:44,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/a4d996227e884d0897e1ebe16b36e1f0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a4d996227e884d0897e1ebe16b36e1f0 2024-11-17T03:14:44,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a4d996227e884d0897e1ebe16b36e1f0, entries=150, sequenceid=378, filesize=12.0 K 2024-11-17T03:14:44,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b29e9d27b06f8df48f1da9682bc65b77 in 214ms, sequenceid=378, compaction requested=true 2024-11-17T03:14:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:44,118 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:44,118 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:44,119 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:44,119 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:44,119 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:44,120 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/dce1e2701ebf469899116b6c1efd0e9c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ed74b22f8fc545a6a676892834827be9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/01697bf16b3640419586e24856c36af3] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.6 K 2024-11-17T03:14:44,120 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:44,120 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:44,120 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:44,120 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting dce1e2701ebf469899116b6c1efd0e9c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731813281844 2024-11-17T03:14:44,120 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7241231d7e28454f91e93d778be67f79, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/69ded74da5d945ed9efdbcbd72a0418c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d44c2d9610f043afb756e947fe040149] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=41.4 K 2024-11-17T03:14:44,121 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ed74b22f8fc545a6a676892834827be9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731813282217 2024-11-17T03:14:44,121 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7241231d7e28454f91e93d778be67f79, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731813281844 2024-11-17T03:14:44,121 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69ded74da5d945ed9efdbcbd72a0418c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731813282217 2024-11-17T03:14:44,122 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d44c2d9610f043afb756e947fe040149, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731813283891 2024-11-17T03:14:44,128 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 01697bf16b3640419586e24856c36af3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731813283893 2024-11-17T03:14:44,143 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:44,143 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d36c32af0cd34b568960ccd995d5386d is 50, key is test_row_0/A:col10/1731813283893/Put/seqid=0 2024-11-17T03:14:44,147 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:44,148 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b72f311646b842c0b46603bd418ead47 is 50, key is test_row_0/B:col10/1731813283893/Put/seqid=0 2024-11-17T03:14:44,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741898_1074 (size=12983) 2024-11-17T03:14:44,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741897_1073 (size=12983) 2024-11-17T03:14:44,181 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/d36c32af0cd34b568960ccd995d5386d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d36c32af0cd34b568960ccd995d5386d 2024-11-17T03:14:44,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T03:14:44,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:44,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:44,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:44,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:44,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:44,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:44,193 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b72f311646b842c0b46603bd418ead47 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b72f311646b842c0b46603bd418ead47 2024-11-17T03:14:44,199 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into d36c32af0cd34b568960ccd995d5386d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:44,199 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:44,199 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=13, startTime=1731813284118; duration=0sec 2024-11-17T03:14:44,201 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:44,201 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:44,201 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:44,206 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:44,206 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:44,206 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:44,206 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/381179b01b7742fd95d8c1e228a5081c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ce6a7c7fdd24430898a3048e1252b045, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a4d996227e884d0897e1ebe16b36e1f0] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.6 K 2024-11-17T03:14:44,207 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into b72f311646b842c0b46603bd418ead47(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:44,208 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:44,208 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=13, startTime=1731813284118; duration=0sec 2024-11-17T03:14:44,209 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:44,209 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:44,210 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381179b01b7742fd95d8c1e228a5081c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731813281844 2024-11-17T03:14:44,212 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6a7c7fdd24430898a3048e1252b045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731813282217 2024-11-17T03:14:44,213 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d996227e884d0897e1ebe16b36e1f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731813283893 2024-11-17T03:14:44,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/ec51e5510ac84c77a0c25b118ae54a18 is 50, key is test_row_0/A:col10/1731813284188/Put/seqid=0 2024-11-17T03:14:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813344222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741899_1075 (size=14741) 2024-11-17T03:14:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813344228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/ec51e5510ac84c77a0c25b118ae54a18 2024-11-17T03:14:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,248 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:44,249 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/cd7761d15e85413cbe8469e253e2ec81 is 50, key is test_row_0/C:col10/1731813283893/Put/seqid=0 2024-11-17T03:14:44,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/92f5b93393004a278c38076d80b07000 is 50, key is test_row_0/B:col10/1731813284188/Put/seqid=0 2024-11-17T03:14:44,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741900_1076 (size=12983) 2024-11-17T03:14:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T03:14:44,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:44,290 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T03:14:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T03:14:44,297 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:44,303 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:44,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:44,311 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/cd7761d15e85413cbe8469e253e2ec81 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/cd7761d15e85413cbe8469e253e2ec81 2024-11-17T03:14:44,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741901_1077 (size=12301) 2024-11-17T03:14:44,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/92f5b93393004a278c38076d80b07000 2024-11-17T03:14:44,327 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into cd7761d15e85413cbe8469e253e2ec81(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:44,327 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:44,327 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=13, startTime=1731813284118; duration=0sec 2024-11-17T03:14:44,327 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:44,327 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:44,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e4c88ab601f34c6c94ab05baa9fa1b03 is 50, key is test_row_0/C:col10/1731813284188/Put/seqid=0 2024-11-17T03:14:44,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813344337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813344347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741902_1078 (size=12301) 2024-11-17T03:14:44,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e4c88ab601f34c6c94ab05baa9fa1b03 2024-11-17T03:14:44,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/ec51e5510ac84c77a0c25b118ae54a18 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ec51e5510ac84c77a0c25b118ae54a18 2024-11-17T03:14:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813344387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813344388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,390 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,389 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ec51e5510ac84c77a0c25b118ae54a18, entries=200, sequenceid=416, filesize=14.4 K 2024-11-17T03:14:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T03:14:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/92f5b93393004a278c38076d80b07000 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/92f5b93393004a278c38076d80b07000 2024-11-17T03:14:44,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813344397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/92f5b93393004a278c38076d80b07000, entries=150, sequenceid=416, filesize=12.0 K 2024-11-17T03:14:44,404 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/e4c88ab601f34c6c94ab05baa9fa1b03 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4c88ab601f34c6c94ab05baa9fa1b03 2024-11-17T03:14:44,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4c88ab601f34c6c94ab05baa9fa1b03, entries=150, sequenceid=416, filesize=12.0 K 2024-11-17T03:14:44,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for b29e9d27b06f8df48f1da9682bc65b77 in 225ms, sequenceid=416, compaction requested=false 2024-11-17T03:14:44,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:44,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T03:14:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:44,470 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T03:14:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:44,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:44,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/566fbbf2e2d542b6a344b4ab73a222a7 is 50, key is test_row_0/A:col10/1731813284224/Put/seqid=0 2024-11-17T03:14:44,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741903_1079 (size=12301) 2024-11-17T03:14:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:44,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T03:14:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813284595, max timestamp=1731813284629Num uniques 859; 2024-11-17T03:14:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813284595, max timestamp=1731813284629Num uniques 859; 2024-11-17T03:14:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813284595, max timestamp=1731813284629Num uniques 859; 2024-11-17T03:14:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813344712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813344717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813344827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813344827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T03:14:44,943 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/566fbbf2e2d542b6a344b4ab73a222a7 2024-11-17T03:14:44,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/55a6489cc9e6412b96eb273d372ec4e7 is 50, key is test_row_0/B:col10/1731813284224/Put/seqid=0 2024-11-17T03:14:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741904_1080 (size=12301) 2024-11-17T03:14:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813345037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813345038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813345347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813345347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,363 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/55a6489cc9e6412b96eb273d372ec4e7 2024-11-17T03:14:45,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/4dcf8587af71410baadbfa7c26a553cd is 50, key is test_row_0/C:col10/1731813284224/Put/seqid=0 2024-11-17T03:14:45,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741905_1081 (size=12301) 2024-11-17T03:14:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T03:14:45,778 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/4dcf8587af71410baadbfa7c26a553cd 2024-11-17T03:14:45,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/566fbbf2e2d542b6a344b4ab73a222a7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/566fbbf2e2d542b6a344b4ab73a222a7 2024-11-17T03:14:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/566fbbf2e2d542b6a344b4ab73a222a7, entries=150, sequenceid=428, filesize=12.0 K 2024-11-17T03:14:45,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/55a6489cc9e6412b96eb273d372ec4e7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/55a6489cc9e6412b96eb273d372ec4e7 2024-11-17T03:14:45,797 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/55a6489cc9e6412b96eb273d372ec4e7, entries=150, sequenceid=428, filesize=12.0 K 2024-11-17T03:14:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/4dcf8587af71410baadbfa7c26a553cd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/4dcf8587af71410baadbfa7c26a553cd 2024-11-17T03:14:45,804 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/4dcf8587af71410baadbfa7c26a553cd, entries=150, sequenceid=428, filesize=12.0 K 2024-11-17T03:14:45,806 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for b29e9d27b06f8df48f1da9682bc65b77 in 1336ms, sequenceid=428, compaction requested=true 2024-11-17T03:14:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T03:14:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T03:14:45,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T03:14:45,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5040 sec 2024-11-17T03:14:45,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5210 sec 2024-11-17T03:14:45,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-17T03:14:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:45,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:45,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:14:45,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:45,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:14:45,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:45,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:14:45,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/e877b414dbcd45afbbcd4e1bcc9aefd1 is 50, key is test_row_0/A:col10/1731813285858/Put/seqid=0 2024-11-17T03:14:45,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741906_1082 (size=14741) 2024-11-17T03:14:45,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/e877b414dbcd45afbbcd4e1bcc9aefd1 2024-11-17T03:14:45,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813345872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813345873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/fbbf1164330e4581b893b202c8116534 is 50, key is test_row_0/B:col10/1731813285858/Put/seqid=0 2024-11-17T03:14:45,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741907_1083 (size=12301) 2024-11-17T03:14:45,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813345987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813345987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813346197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813346197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/fbbf1164330e4581b893b202c8116534 2024-11-17T03:14:46,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/1c43705a7ab54a31bc64d51b92b45ebd is 50, key is test_row_0/C:col10/1731813285858/Put/seqid=0 2024-11-17T03:14:46,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741908_1084 (size=12301) 2024-11-17T03:14:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813346397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813346398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,402 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,403 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813346427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,433 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T03:14:46,438 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:46,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T03:14:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T03:14:46,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:46,441 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:46,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813346507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813346507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T03:14:46,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T03:14:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:46,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:46,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:46,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/1c43705a7ab54a31bc64d51b92b45ebd 2024-11-17T03:14:46,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/e877b414dbcd45afbbcd4e1bcc9aefd1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/e877b414dbcd45afbbcd4e1bcc9aefd1 2024-11-17T03:14:46,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/e877b414dbcd45afbbcd4e1bcc9aefd1, entries=200, sequenceid=467, filesize=14.4 K 2024-11-17T03:14:46,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/fbbf1164330e4581b893b202c8116534 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbbf1164330e4581b893b202c8116534 2024-11-17T03:14:46,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbbf1164330e4581b893b202c8116534, entries=150, sequenceid=467, filesize=12.0 K 2024-11-17T03:14:46,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/1c43705a7ab54a31bc64d51b92b45ebd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1c43705a7ab54a31bc64d51b92b45ebd 2024-11-17T03:14:46,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1c43705a7ab54a31bc64d51b92b45ebd, entries=150, sequenceid=467, filesize=12.0 K 2024-11-17T03:14:46,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for b29e9d27b06f8df48f1da9682bc65b77 in 872ms, sequenceid=467, compaction requested=true 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:46,732 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:46,732 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:46,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:46,733 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:46,733 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:46,734 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:46,734 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:46,734 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:46,734 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:46,734 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d36c32af0cd34b568960ccd995d5386d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ec51e5510ac84c77a0c25b118ae54a18, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/566fbbf2e2d542b6a344b4ab73a222a7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/e877b414dbcd45afbbcd4e1bcc9aefd1] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=53.5 K 2024-11-17T03:14:46,734 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b72f311646b842c0b46603bd418ead47, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/92f5b93393004a278c38076d80b07000, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/55a6489cc9e6412b96eb273d372ec4e7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbbf1164330e4581b893b202c8116534] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=48.7 K 2024-11-17T03:14:46,735 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b72f311646b842c0b46603bd418ead47, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731813283893 2024-11-17T03:14:46,735 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d36c32af0cd34b568960ccd995d5386d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731813283893 2024-11-17T03:14:46,735 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec51e5510ac84c77a0c25b118ae54a18, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731813284027 2024-11-17T03:14:46,735 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f5b93393004a278c38076d80b07000, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731813284027 2024-11-17T03:14:46,735 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566fbbf2e2d542b6a344b4ab73a222a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731813284214 2024-11-17T03:14:46,736 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a6489cc9e6412b96eb273d372ec4e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731813284214 2024-11-17T03:14:46,736 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e877b414dbcd45afbbcd4e1bcc9aefd1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731813284696 2024-11-17T03:14:46,736 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting fbbf1164330e4581b893b202c8116534, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731813284711 2024-11-17T03:14:46,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:46,749 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:46,749 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#73 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:46,750 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/8e771394e711461ba59f2e2c4b6b4893 is 50, key is test_row_0/B:col10/1731813285858/Put/seqid=0 2024-11-17T03:14:46,753 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#72 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:46,753 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/45eba99ab8d04c08b30ec845d6ffc0db is 50, key is test_row_0/A:col10/1731813285858/Put/seqid=0 2024-11-17T03:14:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/c2a3f74a07ba487880fff26ac4121f91 is 50, key is test_row_0/A:col10/1731813285871/Put/seqid=0 2024-11-17T03:14:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T03:14:46,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741909_1085 (size=13119) 2024-11-17T03:14:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741911_1087 (size=13119) 2024-11-17T03:14:46,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741910_1086 (size=9857) 2024-11-17T03:14:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:47,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:47,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813287038, max timestamp=1731813287061Num uniques 859; 2024-11-17T03:14:47,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:14:47,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813287038, max timestamp=1731813287061Num uniques 859; 2024-11-17T03:14:47,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813287038, max timestamp=1731813287061Num uniques 859; 2024-11-17T03:14:47,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:14:47,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:14:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T03:14:47,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813347108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:47,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731813287099, max timestamp=1731813287109Num uniques 1168; 2024-11-17T03:14:47,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731813287099, max timestamp=1731813287109Num uniques 1168; 2024-11-17T03:14:47,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-11-17T03:14:47,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-11-17T03:14:47,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731813287099, max timestamp=1731813287109Num uniques 1168; 2024-11-17T03:14:47,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=7688, suffix heap size=149440,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=168 2024-11-17T03:14:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:47,168 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/45eba99ab8d04c08b30ec845d6ffc0db as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/45eba99ab8d04c08b30ec845d6ffc0db 2024-11-17T03:14:47,174 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/8e771394e711461ba59f2e2c4b6b4893 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/8e771394e711461ba59f2e2c4b6b4893 2024-11-17T03:14:47,175 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into 45eba99ab8d04c08b30ec845d6ffc0db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:47,175 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:47,175 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=12, startTime=1731813286732; duration=0sec 2024-11-17T03:14:47,175 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:47,175 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:47,175 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:47,176 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:47,176 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/c2a3f74a07ba487880fff26ac4121f91 2024-11-17T03:14:47,177 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:47,177 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:47,177 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/cd7761d15e85413cbe8469e253e2ec81, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4c88ab601f34c6c94ab05baa9fa1b03, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/4dcf8587af71410baadbfa7c26a553cd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1c43705a7ab54a31bc64d51b92b45ebd] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=48.7 K 2024-11-17T03:14:47,178 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd7761d15e85413cbe8469e253e2ec81, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731813283893 2024-11-17T03:14:47,179 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c88ab601f34c6c94ab05baa9fa1b03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731813284027 2024-11-17T03:14:47,180 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dcf8587af71410baadbfa7c26a553cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731813284214 2024-11-17T03:14:47,181 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c43705a7ab54a31bc64d51b92b45ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731813284711 2024-11-17T03:14:47,183 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into 8e771394e711461ba59f2e2c4b6b4893(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:47,184 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:47,184 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=12, startTime=1731813286732; duration=0sec 2024-11-17T03:14:47,184 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:47,184 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:47,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d is 50, key is test_row_0/B:col10/1731813285871/Put/seqid=0 2024-11-17T03:14:47,198 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:47,198 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/f129310823ed4ffb809fe6343996cbc3 is 50, key is test_row_0/C:col10/1731813285858/Put/seqid=0 2024-11-17T03:14:47,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741912_1088 (size=9857) 2024-11-17T03:14:47,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741913_1089 (size=13119) 2024-11-17T03:14:47,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813287163, max timestamp=1731813287210Num uniques 1027; 2024-11-17T03:14:47,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T03:14:47,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813287163, max timestamp=1731813287210Num uniques 1027; 2024-11-17T03:14:47,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T03:14:47,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813287163, max timestamp=1731813287210Num uniques 1027; 2024-11-17T03:14:47,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T03:14:47,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731813287238, max timestamp=1731813287275Num uniques 1036; 2024-11-17T03:14:47,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731813287238, max timestamp=1731813287275Num uniques 1036; 2024-11-17T03:14:47,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T03:14:47,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T03:14:47,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731813287238, max timestamp=1731813287275Num uniques 1036; 2024-11-17T03:14:47,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T03:14:47,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731813287295, max timestamp=1731813287327Num uniques 1045; 2024-11-17T03:14:47,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T03:14:47,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731813287295, max timestamp=1731813287327Num uniques 1045; 2024-11-17T03:14:47,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T03:14:47,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731813287295, max timestamp=1731813287327Num uniques 1045; 2024-11-17T03:14:47,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T03:14:47,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731813287369, max timestamp=1731813287382Num uniques 1104; 2024-11-17T03:14:47,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731813287369, max timestamp=1731813287382Num uniques 1104; 2024-11-17T03:14:47,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731813287369, max timestamp=1731813287382Num uniques 1104; 2024-11-17T03:14:47,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-17T03:14:47,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-17T03:14:47,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-17T03:14:47,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731813287410, max timestamp=1731813287426Num uniques 1063; 2024-11-17T03:14:47,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T03:14:47,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731813287410, max timestamp=1731813287426Num uniques 1063; 2024-11-17T03:14:47,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T03:14:47,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731813287410, max timestamp=1731813287426Num uniques 1063; 2024-11-17T03:14:47,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T03:14:47,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731813287459, max timestamp=1731813287478Num uniques 1072; 2024-11-17T03:14:47,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731813287459, max timestamp=1731813287478Num uniques 1072; 2024-11-17T03:14:47,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-17T03:14:47,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-17T03:14:47,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731813287459, max timestamp=1731813287478Num uniques 1072; 2024-11-17T03:14:47,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-17T03:14:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731813287510, max timestamp=1731813287538Num uniques 1131; 2024-11-17T03:14:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-17T03:14:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731813287510, max timestamp=1731813287538Num uniques 1131; 2024-11-17T03:14:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731813287510, max timestamp=1731813287538Num uniques 1131; 2024-11-17T03:14:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-17T03:14:47,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-17T03:14:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T03:14:47,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731813287577, max timestamp=1731813287595Num uniques 1140; 2024-11-17T03:14:47,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731813287577, max timestamp=1731813287595Num uniques 1140; 2024-11-17T03:14:47,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=13280, suffix heap size=146528,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=290 2024-11-17T03:14:47,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=13280, suffix heap size=146528,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=290 2024-11-17T03:14:47,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731813287577, max timestamp=1731813287595Num uniques 1140; 2024-11-17T03:14:47,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=13280, suffix heap size=146528,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=290 2024-11-17T03:14:47,602 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d 2024-11-17T03:14:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/81b2054f522c4cf283ce51f7de7f023d is 50, key is test_row_0/C:col10/1731813285871/Put/seqid=0 2024-11-17T03:14:47,619 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/f129310823ed4ffb809fe6343996cbc3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/f129310823ed4ffb809fe6343996cbc3 2024-11-17T03:14:47,628 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into f129310823ed4ffb809fe6343996cbc3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:47,628 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:47,628 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=12, startTime=1731813286732; duration=0sec 2024-11-17T03:14:47,628 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:47,628 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:47,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741914_1090 (size=9857) 2024-11-17T03:14:47,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731813287630, max timestamp=1731813287648Num uniques 1149; 2024-11-17T03:14:47,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-17T03:14:47,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731813287630, max timestamp=1731813287648Num uniques 1149; 2024-11-17T03:14:47,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-17T03:14:47,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731813287630, max timestamp=1731813287648Num uniques 1149; 2024-11-17T03:14:47,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=11404, suffix heap size=147464,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=249 2024-11-17T03:14:47,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731813287685, max timestamp=1731813287699Num uniques 1108; 2024-11-17T03:14:47,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-11-17T03:14:47,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731813287685, max timestamp=1731813287699Num uniques 1108; 2024-11-17T03:14:47,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-11-17T03:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731813287685, max timestamp=1731813287699Num uniques 1108; 2024-11-17T03:14:47,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=11810, suffix heap size=143200,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=258 2024-11-17T03:14:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813287724, max timestamp=1731813287743Num uniques 1117; 2024-11-17T03:14:47,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-17T03:14:47,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813287724, max timestamp=1731813287743Num uniques 1117; 2024-11-17T03:14:47,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813287724, max timestamp=1731813287743Num uniques 1117; 2024-11-17T03:14:47,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-17T03:14:47,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-17T03:14:47,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731813287768, max timestamp=1731813287784Num uniques 1076; 2024-11-17T03:14:47,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-17T03:14:47,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731813287768, max timestamp=1731813287784Num uniques 1076; 2024-11-17T03:14:47,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-17T03:14:47,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731813287768, max timestamp=1731813287784Num uniques 1076; 2024-11-17T03:14:47,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-17T03:14:47,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1731813287791, max timestamp=1731813287862Num uniques 1085; 2024-11-17T03:14:47,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1731813287791, max timestamp=1731813287862Num uniques 1085; 2024-11-17T03:14:47,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1731813287791, max timestamp=1731813287862Num uniques 1085; 2024-11-17T03:14:47,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-11-17T03:14:47,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-11-17T03:14:47,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-11-17T03:14:47,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813347899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:47,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731813287881, max timestamp=1731813287901Num uniques 1194; 2024-11-17T03:14:47,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54684, new segment data size=13464, suffix heap size=152144,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1194, new segment cells count=294 2024-11-17T03:14:47,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731813287881, max timestamp=1731813287901Num uniques 1194; 2024-11-17T03:14:47,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54684, new segment data size=13464, suffix heap size=152144,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1194, new segment cells count=294 2024-11-17T03:14:47,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1731813287881, max timestamp=1731813287901Num uniques 1194; 2024-11-17T03:14:47,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54684, new segment data size=13464, suffix heap size=152144,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1194, new segment cells count=294 2024-11-17T03:14:47,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813347974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813348007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,035 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/81b2054f522c4cf283ce51f7de7f023d 2024-11-17T03:14:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/c2a3f74a07ba487880fff26ac4121f91 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/c2a3f74a07ba487880fff26ac4121f91 2024-11-17T03:14:48,048 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/c2a3f74a07ba487880fff26ac4121f91, entries=100, sequenceid=474, filesize=9.6 K 2024-11-17T03:14:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d 2024-11-17T03:14:48,056 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d, entries=100, sequenceid=474, filesize=9.6 K 2024-11-17T03:14:48,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/81b2054f522c4cf283ce51f7de7f023d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/81b2054f522c4cf283ce51f7de7f023d 2024-11-17T03:14:48,063 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/81b2054f522c4cf283ce51f7de7f023d, entries=100, sequenceid=474, filesize=9.6 K 2024-11-17T03:14:48,065 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for b29e9d27b06f8df48f1da9682bc65b77 in 1315ms, sequenceid=474, compaction requested=false 2024-11-17T03:14:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T03:14:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T03:14:48,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T03:14:48,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-17T03:14:48,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6340 sec 2024-11-17T03:14:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:48,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=161.02 KB heapSize=452.72 KB 2024-11-17T03:14:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:14:48,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:48,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:14:48,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/ecbf8d81f1df40d4a6f80aa3002d792a is 50, key is test_row_0/A:col10/1731813287973/Put/seqid=0 2024-11-17T03:14:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741915_1091 (size=14741) 2024-11-17T03:14:48,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813348103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813348206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813348217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813348417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/ecbf8d81f1df40d4a6f80aa3002d792a 2024-11-17T03:14:48,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a2bf8e94b2904604924256d373472093 is 50, key is test_row_0/B:col10/1731813287973/Put/seqid=0 2024-11-17T03:14:48,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813348527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741916_1092 (size=12301) 2024-11-17T03:14:48,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a2bf8e94b2904604924256d373472093 2024-11-17T03:14:48,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/6fb233febcee475bb44a9a25a3c4fa45 is 50, key is test_row_0/C:col10/1731813287973/Put/seqid=0 2024-11-17T03:14:48,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741917_1093 (size=12301) 2024-11-17T03:14:48,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=773 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/6fb233febcee475bb44a9a25a3c4fa45 2024-11-17T03:14:48,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/ecbf8d81f1df40d4a6f80aa3002d792a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ecbf8d81f1df40d4a6f80aa3002d792a 2024-11-17T03:14:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T03:14:48,588 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:48,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:48,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ecbf8d81f1df40d4a6f80aa3002d792a, entries=200, sequenceid=773, filesize=14.4 K 2024-11-17T03:14:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T03:14:48,592 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T03:14:48,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:48,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/a2bf8e94b2904604924256d373472093 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a2bf8e94b2904604924256d373472093 2024-11-17T03:14:48,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:48,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a2bf8e94b2904604924256d373472093, entries=150, sequenceid=773, filesize=12.0 K 2024-11-17T03:14:48,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/6fb233febcee475bb44a9a25a3c4fa45 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6fb233febcee475bb44a9a25a3c4fa45 2024-11-17T03:14:48,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6fb233febcee475bb44a9a25a3c4fa45, entries=150, sequenceid=773, filesize=12.0 K 2024-11-17T03:14:48,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~452.67 KB/463536, currentSize=20.13 KB/20610 for b29e9d27b06f8df48f1da9682bc65b77 in 534ms, sequenceid=773, compaction requested=true 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:48,623 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:48,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T03:14:48,624 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:48,625 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:48,625 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:48,625 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,626 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/45eba99ab8d04c08b30ec845d6ffc0db, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/c2a3f74a07ba487880fff26ac4121f91, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ecbf8d81f1df40d4a6f80aa3002d792a] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=36.8 K 2024-11-17T03:14:48,626 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:48,626 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:48,626 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,626 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/8e771394e711461ba59f2e2c4b6b4893, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a2bf8e94b2904604924256d373472093] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=34.5 K 2024-11-17T03:14:48,627 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 45eba99ab8d04c08b30ec845d6ffc0db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731813284711 2024-11-17T03:14:48,627 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e771394e711461ba59f2e2c4b6b4893, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731813284711 2024-11-17T03:14:48,628 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a3f74a07ba487880fff26ac4121f91, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731813285869 2024-11-17T03:14:48,631 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae61f17cf85c4bc5b7f4ab5fd98d3f9d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731813285869 2024-11-17T03:14:48,631 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ecbf8d81f1df40d4a6f80aa3002d792a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1731813287946 2024-11-17T03:14:48,631 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2bf8e94b2904604924256d373472093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1731813287954 2024-11-17T03:14:48,647 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:48,647 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/73ede981602544569217ed6e0d167e9c is 50, key is test_row_0/B:col10/1731813287973/Put/seqid=0 2024-11-17T03:14:48,654 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:48,655 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/4299640812bf4aac86070167da1b9e12 is 50, key is test_row_0/A:col10/1731813287973/Put/seqid=0 2024-11-17T03:14:48,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741918_1094 (size=13221) 2024-11-17T03:14:48,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741919_1095 (size=13221) 2024-11-17T03:14:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T03:14:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:48,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:48,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:48,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/050774fdb7734872b67b96637ac4f243 is 50, key is test_row_0/A:col10/1731813288736/Put/seqid=0 2024-11-17T03:14:48,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:48,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741920_1096 (size=17181) 2024-11-17T03:14:48,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813288787, max timestamp=1731813288816Num uniques 859; 2024-11-17T03:14:48,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:48,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813288787, max timestamp=1731813288816Num uniques 859; 2024-11-17T03:14:48,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:48,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813288787, max timestamp=1731813288816Num uniques 859; 2024-11-17T03:14:48,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:14:48,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813348882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:48,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:48,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:48,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:48,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:48,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T03:14:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813348997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813349037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:49,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,072 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/73ede981602544569217ed6e0d167e9c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/73ede981602544569217ed6e0d167e9c 2024-11-17T03:14:49,079 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/4299640812bf4aac86070167da1b9e12 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4299640812bf4aac86070167da1b9e12 2024-11-17T03:14:49,082 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into 73ede981602544569217ed6e0d167e9c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:49,082 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:49,082 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=13, startTime=1731813288623; duration=0sec 2024-11-17T03:14:49,083 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:49,083 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:49,083 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:49,084 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:49,084 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:49,084 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,084 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/f129310823ed4ffb809fe6343996cbc3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/81b2054f522c4cf283ce51f7de7f023d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6fb233febcee475bb44a9a25a3c4fa45] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=34.5 K 2024-11-17T03:14:49,085 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f129310823ed4ffb809fe6343996cbc3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731813284711 2024-11-17T03:14:49,085 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b2054f522c4cf283ce51f7de7f023d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731813285869 2024-11-17T03:14:49,085 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into 4299640812bf4aac86070167da1b9e12(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:49,086 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:49,086 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fb233febcee475bb44a9a25a3c4fa45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1731813287954 2024-11-17T03:14:49,086 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=13, startTime=1731813288623; duration=0sec 2024-11-17T03:14:49,086 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:49,086 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:49,095 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:49,095 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/63940ede9b464001bfb4f0d44ca79bfc is 50, key is test_row_0/C:col10/1731813287973/Put/seqid=0 2024-11-17T03:14:49,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741921_1097 (size=13221) 2024-11-17T03:14:49,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/050774fdb7734872b67b96637ac4f243 2024-11-17T03:14:49,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/5e80dd8bcb6641ebbf0a7562cb47429e is 50, key is test_row_0/B:col10/1731813288736/Put/seqid=0 2024-11-17T03:14:49,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741922_1098 (size=12301) 2024-11-17T03:14:49,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/5e80dd8bcb6641ebbf0a7562cb47429e 2024-11-17T03:14:49,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/10359b7e7cf84a01baa298c8a110e239 is 50, key is test_row_0/C:col10/1731813288736/Put/seqid=0 2024-11-17T03:14:49,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741923_1099 (size=12301) 2024-11-17T03:14:49,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813349207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T03:14:49,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,510 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/63940ede9b464001bfb4f0d44ca79bfc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/63940ede9b464001bfb4f0d44ca79bfc 2024-11-17T03:14:49,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:49,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813349517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:49,523 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into 63940ede9b464001bfb4f0d44ca79bfc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:49,523 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:49,523 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=13, startTime=1731813288623; duration=0sec 2024-11-17T03:14:49,523 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:49,524 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:49,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=783 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/10359b7e7cf84a01baa298c8a110e239 2024-11-17T03:14:49,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/050774fdb7734872b67b96637ac4f243 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/050774fdb7734872b67b96637ac4f243 2024-11-17T03:14:49,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/050774fdb7734872b67b96637ac4f243, entries=250, sequenceid=783, filesize=16.8 K 2024-11-17T03:14:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/5e80dd8bcb6641ebbf0a7562cb47429e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5e80dd8bcb6641ebbf0a7562cb47429e 2024-11-17T03:14:49,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5e80dd8bcb6641ebbf0a7562cb47429e, entries=150, sequenceid=783, filesize=12.0 K 2024-11-17T03:14:49,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/10359b7e7cf84a01baa298c8a110e239 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/10359b7e7cf84a01baa298c8a110e239 2024-11-17T03:14:49,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/10359b7e7cf84a01baa298c8a110e239, entries=150, sequenceid=783, filesize=12.0 K 2024-11-17T03:14:49,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b29e9d27b06f8df48f1da9682bc65b77 in 909ms, sequenceid=783, compaction requested=false 2024-11-17T03:14:49,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:49,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T03:14:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,670 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T03:14:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:49,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a80790f70ea44838915f888ec9e98bde is 50, key is test_row_0/A:col10/1731813288870/Put/seqid=0 2024-11-17T03:14:49,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741924_1100 (size=12301) 2024-11-17T03:14:49,690 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=822 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a80790f70ea44838915f888ec9e98bde 2024-11-17T03:14:49,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3a7abda1c8f4745b0831e0dc56a1514 is 50, key is test_row_0/B:col10/1731813288870/Put/seqid=0 2024-11-17T03:14:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741925_1101 (size=12301) 2024-11-17T03:14:49,711 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=822 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3a7abda1c8f4745b0831e0dc56a1514 2024-11-17T03:14:49,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/af66a0cb6b434b8db0a0925dd993d505 is 50, key is test_row_0/C:col10/1731813288870/Put/seqid=0 2024-11-17T03:14:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T03:14:49,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741926_1102 (size=12301) 2024-11-17T03:14:49,736 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=822 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/af66a0cb6b434b8db0a0925dd993d505 2024-11-17T03:14:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/a80790f70ea44838915f888ec9e98bde as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a80790f70ea44838915f888ec9e98bde 2024-11-17T03:14:49,757 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a80790f70ea44838915f888ec9e98bde, entries=150, sequenceid=822, filesize=12.0 K 2024-11-17T03:14:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3a7abda1c8f4745b0831e0dc56a1514 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3a7abda1c8f4745b0831e0dc56a1514 2024-11-17T03:14:49,765 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3a7abda1c8f4745b0831e0dc56a1514, entries=150, sequenceid=822, filesize=12.0 K 2024-11-17T03:14:49,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/af66a0cb6b434b8db0a0925dd993d505 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/af66a0cb6b434b8db0a0925dd993d505 2024-11-17T03:14:49,774 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/af66a0cb6b434b8db0a0925dd993d505, entries=150, sequenceid=822, filesize=12.0 K 2024-11-17T03:14:49,775 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for b29e9d27b06f8df48f1da9682bc65b77 in 105ms, sequenceid=822, compaction requested=true 2024-11-17T03:14:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:49,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T03:14:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T03:14:49,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T03:14:49,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1840 sec 2024-11-17T03:14:49,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1940 sec 2024-11-17T03:14:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:50,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:14:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:50,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/3e3520e7c9d8490f89ca93ac6fe13d97 is 50, key is test_row_0/A:col10/1731813290040/Put/seqid=0 2024-11-17T03:14:50,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741927_1103 (size=14741) 2024-11-17T03:14:50,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813290078, max timestamp=1731813290094Num uniques 859; 2024-11-17T03:14:50,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:14:50,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813290078, max timestamp=1731813290094Num uniques 859; 2024-11-17T03:14:50,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813290078, max timestamp=1731813290094Num uniques 859; 2024-11-17T03:14:50,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:14:50,097 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:14:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813350139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813350141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813350247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813350247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36014 deadline: 1731813350416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,417 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35942 deadline: 1731813350437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,445 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813350457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813350457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/3e3520e7c9d8490f89ca93ac6fe13d97 2024-11-17T03:14:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35940 deadline: 1731813350467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,475 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in b29e9d27b06f8df48f1da9682bc65b77 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:14:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bdddb4d1d5614456b95a673797de9aa9 is 50, key is test_row_0/B:col10/1731813290040/Put/seqid=0 2024-11-17T03:14:50,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741928_1104 (size=12301) 2024-11-17T03:14:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T03:14:50,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:50,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T03:14:50,739 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T03:14:50,740 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:50,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813350767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813350767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T03:14:50,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bdddb4d1d5614456b95a673797de9aa9 2024-11-17T03:14:50,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T03:14:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:50,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:50,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:50,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ae95d29798d946fe8726aa794ba53ad1 is 50, key is test_row_0/C:col10/1731813290040/Put/seqid=0 2024-11-17T03:14:50,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741929_1105 (size=12301) 2024-11-17T03:14:51,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T03:14:51,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:51,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T03:14:51,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T03:14:51,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813351277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:51,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813351277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:51,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:51,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=832 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ae95d29798d946fe8726aa794ba53ad1 2024-11-17T03:14:51,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/3e3520e7c9d8490f89ca93ac6fe13d97 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/3e3520e7c9d8490f89ca93ac6fe13d97 2024-11-17T03:14:51,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/3e3520e7c9d8490f89ca93ac6fe13d97, entries=200, sequenceid=832, filesize=14.4 K 2024-11-17T03:14:51,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/bdddb4d1d5614456b95a673797de9aa9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bdddb4d1d5614456b95a673797de9aa9 2024-11-17T03:14:51,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bdddb4d1d5614456b95a673797de9aa9, entries=150, sequenceid=832, filesize=12.0 K 2024-11-17T03:14:51,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/ae95d29798d946fe8726aa794ba53ad1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ae95d29798d946fe8726aa794ba53ad1 2024-11-17T03:14:51,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ae95d29798d946fe8726aa794ba53ad1, entries=150, sequenceid=832, filesize=12.0 K 2024-11-17T03:14:51,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b29e9d27b06f8df48f1da9682bc65b77 in 1300ms, sequenceid=832, compaction requested=true 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:51,350 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:51,350 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:51,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:51,352 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:51,352 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:51,352 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:51,352 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:51,352 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,352 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,353 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/73ede981602544569217ed6e0d167e9c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5e80dd8bcb6641ebbf0a7562cb47429e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3a7abda1c8f4745b0831e0dc56a1514, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bdddb4d1d5614456b95a673797de9aa9] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=48.9 K 2024-11-17T03:14:51,353 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4299640812bf4aac86070167da1b9e12, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/050774fdb7734872b67b96637ac4f243, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a80790f70ea44838915f888ec9e98bde, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/3e3520e7c9d8490f89ca93ac6fe13d97] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=56.1 K 2024-11-17T03:14:51,353 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4299640812bf4aac86070167da1b9e12, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1731813287954 2024-11-17T03:14:51,353 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ede981602544569217ed6e0d167e9c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1731813287954 2024-11-17T03:14:51,354 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050774fdb7734872b67b96637ac4f243, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731813288098 2024-11-17T03:14:51,354 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e80dd8bcb6641ebbf0a7562cb47429e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731813288728 2024-11-17T03:14:51,354 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80790f70ea44838915f888ec9e98bde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=822, earliestPutTs=1731813288870 2024-11-17T03:14:51,354 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a7abda1c8f4745b0831e0dc56a1514, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=822, earliestPutTs=1731813288870 2024-11-17T03:14:51,355 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3520e7c9d8490f89ca93ac6fe13d97, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=832, earliestPutTs=1731813290037 2024-11-17T03:14:51,355 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting bdddb4d1d5614456b95a673797de9aa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=832, earliestPutTs=1731813290037 2024-11-17T03:14:51,370 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#93 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:51,370 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/aba477694cc54dff9bbd818342e1a2cc is 50, key is test_row_0/A:col10/1731813290040/Put/seqid=0 2024-11-17T03:14:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T03:14:51,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T03:14:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,375 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-17T03:14:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:51,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:14:51,384 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#94 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:51,385 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/979ad4df2de742bea0929f346c87403c is 50, key is test_row_0/B:col10/1731813290040/Put/seqid=0 2024-11-17T03:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/8e9d2d11985e4926a4098fdd8d7f8d1c is 50, key is test_row_0/A:col10/1731813290127/Put/seqid=0 2024-11-17T03:14:51,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741931_1107 (size=13357) 2024-11-17T03:14:51,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741930_1106 (size=13357) 2024-11-17T03:14:51,444 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/979ad4df2de742bea0929f346c87403c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/979ad4df2de742bea0929f346c87403c 2024-11-17T03:14:51,452 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/aba477694cc54dff9bbd818342e1a2cc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/aba477694cc54dff9bbd818342e1a2cc 2024-11-17T03:14:51,460 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into 979ad4df2de742bea0929f346c87403c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:51,460 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:51,460 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=12, startTime=1731813291350; duration=0sec 2024-11-17T03:14:51,461 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:51,461 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:51,461 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:14:51,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741932_1108 (size=12301) 2024-11-17T03:14:51,463 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:14:51,463 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:51,463 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:51,463 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/63940ede9b464001bfb4f0d44ca79bfc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/10359b7e7cf84a01baa298c8a110e239, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/af66a0cb6b434b8db0a0925dd993d505, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ae95d29798d946fe8726aa794ba53ad1] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=48.9 K 2024-11-17T03:14:51,464 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 63940ede9b464001bfb4f0d44ca79bfc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=773, earliestPutTs=1731813287954 2024-11-17T03:14:51,465 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=867 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/8e9d2d11985e4926a4098fdd8d7f8d1c 2024-11-17T03:14:51,466 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 10359b7e7cf84a01baa298c8a110e239, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=783, earliestPutTs=1731813288728 2024-11-17T03:14:51,466 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting af66a0cb6b434b8db0a0925dd993d505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=822, earliestPutTs=1731813288870 2024-11-17T03:14:51,467 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ae95d29798d946fe8726aa794ba53ad1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=832, earliestPutTs=1731813290037 2024-11-17T03:14:51,469 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into aba477694cc54dff9bbd818342e1a2cc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:51,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:51,470 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=12, startTime=1731813291350; duration=0sec 2024-11-17T03:14:51,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:51,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:51,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/cda13c2bdbf049a7a3ab2e9bbc991ee5 is 50, key is test_row_0/B:col10/1731813290127/Put/seqid=0 2024-11-17T03:14:51,503 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:51,503 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/99c7125866e4455d887c241c038d7194 is 50, key is test_row_0/C:col10/1731813290040/Put/seqid=0 2024-11-17T03:14:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741933_1109 (size=12301) 2024-11-17T03:14:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741934_1110 (size=13357) 2024-11-17T03:14:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T03:14:51,913 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=867 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/cda13c2bdbf049a7a3ab2e9bbc991ee5 2024-11-17T03:14:51,925 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/99c7125866e4455d887c241c038d7194 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/99c7125866e4455d887c241c038d7194 2024-11-17T03:14:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/54bd9b8944904eaaa224303ccff5fa25 is 50, key is test_row_0/C:col10/1731813290127/Put/seqid=0 2024-11-17T03:14:51,934 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into 99c7125866e4455d887c241c038d7194(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:51,934 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:51,934 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=12, startTime=1731813291350; duration=0sec 2024-11-17T03:14:51,935 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:51,935 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:51,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741935_1111 (size=12301) 2024-11-17T03:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:52,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 369 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813352305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813352306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,338 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=867 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/54bd9b8944904eaaa224303ccff5fa25 2024-11-17T03:14:52,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/8e9d2d11985e4926a4098fdd8d7f8d1c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8e9d2d11985e4926a4098fdd8d7f8d1c 2024-11-17T03:14:52,349 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8e9d2d11985e4926a4098fdd8d7f8d1c, entries=150, sequenceid=867, filesize=12.0 K 2024-11-17T03:14:52,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/cda13c2bdbf049a7a3ab2e9bbc991ee5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cda13c2bdbf049a7a3ab2e9bbc991ee5 2024-11-17T03:14:52,357 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cda13c2bdbf049a7a3ab2e9bbc991ee5, entries=150, sequenceid=867, filesize=12.0 K 2024-11-17T03:14:52,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/54bd9b8944904eaaa224303ccff5fa25 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/54bd9b8944904eaaa224303ccff5fa25 2024-11-17T03:14:52,363 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/54bd9b8944904eaaa224303ccff5fa25, entries=150, sequenceid=867, filesize=12.0 K 2024-11-17T03:14:52,364 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for b29e9d27b06f8df48f1da9682bc65b77 in 989ms, sequenceid=867, compaction requested=false 2024-11-17T03:14:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T03:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T03:14:52,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T03:14:52,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6260 sec 2024-11-17T03:14:52,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6310 sec 2024-11-17T03:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:52,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:14:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:52,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:52,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/b996019ed3da4e00bcebd47f978fa709 is 50, key is test_row_0/A:col10/1731813292417/Put/seqid=0 2024-11-17T03:14:52,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741936_1112 (size=17181) 2024-11-17T03:14:52,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813352447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 380 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813352449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813352557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 381 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813352557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813352766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813352767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:52,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/b996019ed3da4e00bcebd47f978fa709 2024-11-17T03:14:52,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/e678f006be3e4b31802be2e252c4f072 is 50, key is test_row_0/B:col10/1731813292417/Put/seqid=0 2024-11-17T03:14:52,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741937_1113 (size=12301) 2024-11-17T03:14:52,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/e678f006be3e4b31802be2e252c4f072 2024-11-17T03:14:52,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/23f8f834df9f47d3bcdf99baa8e445a2 is 50, key is test_row_0/C:col10/1731813292417/Put/seqid=0 2024-11-17T03:14:52,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741938_1114 (size=12301) 2024-11-17T03:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T03:14:52,887 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:14:52,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:14:52,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T03:14:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T03:14:52,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:14:52,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:14:52,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:14:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T03:14:53,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T03:14:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35998 deadline: 1731813353077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:53,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35960 deadline: 1731813353077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:14:53,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:14:53,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T03:14:53,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:53,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:53,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:14:53,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T03:14:53,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/23f8f834df9f47d3bcdf99baa8e445a2 2024-11-17T03:14:53,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/b996019ed3da4e00bcebd47f978fa709 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b996019ed3da4e00bcebd47f978fa709 2024-11-17T03:14:53,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b996019ed3da4e00bcebd47f978fa709, entries=250, sequenceid=882, filesize=16.8 K 2024-11-17T03:14:53,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/e678f006be3e4b31802be2e252c4f072 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e678f006be3e4b31802be2e252c4f072 2024-11-17T03:14:53,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e678f006be3e4b31802be2e252c4f072, entries=150, sequenceid=882, filesize=12.0 K 2024-11-17T03:14:53,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/23f8f834df9f47d3bcdf99baa8e445a2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/23f8f834df9f47d3bcdf99baa8e445a2 2024-11-17T03:14:53,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/23f8f834df9f47d3bcdf99baa8e445a2, entries=150, sequenceid=882, filesize=12.0 K 2024-11-17T03:14:53,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b29e9d27b06f8df48f1da9682bc65b77 in 861ms, sequenceid=882, compaction requested=true 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:53,279 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:53,279 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b29e9d27b06f8df48f1da9682bc65b77:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:14:53,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:53,280 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:53,280 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:53,280 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/A is initiating minor compaction (all files) 2024-11-17T03:14:53,280 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/B is initiating minor compaction (all files) 2024-11-17T03:14:53,281 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/A in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,281 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/B in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,281 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/aba477694cc54dff9bbd818342e1a2cc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8e9d2d11985e4926a4098fdd8d7f8d1c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b996019ed3da4e00bcebd47f978fa709] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=41.8 K 2024-11-17T03:14:53,281 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/979ad4df2de742bea0929f346c87403c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cda13c2bdbf049a7a3ab2e9bbc991ee5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e678f006be3e4b31802be2e252c4f072] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=37.1 K 2024-11-17T03:14:53,281 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba477694cc54dff9bbd818342e1a2cc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=832, earliestPutTs=1731813290037 2024-11-17T03:14:53,281 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 979ad4df2de742bea0929f346c87403c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=832, earliestPutTs=1731813290037 2024-11-17T03:14:53,281 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9d2d11985e4926a4098fdd8d7f8d1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=867, earliestPutTs=1731813290127 2024-11-17T03:14:53,281 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting cda13c2bdbf049a7a3ab2e9bbc991ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=867, earliestPutTs=1731813290127 2024-11-17T03:14:53,282 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e678f006be3e4b31802be2e252c4f072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1731813292305 2024-11-17T03:14:53,282 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b996019ed3da4e00bcebd47f978fa709, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1731813292299 2024-11-17T03:14:53,289 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:53,290 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/e672a5be031d4dd69fafa9e432b7fd4a is 50, key is test_row_0/B:col10/1731813292417/Put/seqid=0 2024-11-17T03:14:53,293 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:53,294 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/8756026b3dcf478fad64eacafd71ae14 is 50, key is test_row_0/A:col10/1731813292417/Put/seqid=0 2024-11-17T03:14:53,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741939_1115 (size=13459) 2024-11-17T03:14:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741940_1116 (size=13459) 2024-11-17T03:14:53,307 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/e672a5be031d4dd69fafa9e432b7fd4a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e672a5be031d4dd69fafa9e432b7fd4a 2024-11-17T03:14:53,314 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/B of b29e9d27b06f8df48f1da9682bc65b77 into e672a5be031d4dd69fafa9e432b7fd4a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:53,314 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/8756026b3dcf478fad64eacafd71ae14 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8756026b3dcf478fad64eacafd71ae14 2024-11-17T03:14:53,314 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:53,314 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/B, priority=13, startTime=1731813293279; duration=0sec 2024-11-17T03:14:53,314 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:14:53,315 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:B 2024-11-17T03:14:53,315 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:14:53,316 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:14:53,317 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): b29e9d27b06f8df48f1da9682bc65b77/C is initiating minor compaction (all files) 2024-11-17T03:14:53,318 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b29e9d27b06f8df48f1da9682bc65b77/C in TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,318 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/99c7125866e4455d887c241c038d7194, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/54bd9b8944904eaaa224303ccff5fa25, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/23f8f834df9f47d3bcdf99baa8e445a2] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp, totalSize=37.1 K 2024-11-17T03:14:53,319 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c7125866e4455d887c241c038d7194, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=832, earliestPutTs=1731813290037 2024-11-17T03:14:53,320 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 54bd9b8944904eaaa224303ccff5fa25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=867, earliestPutTs=1731813290127 2024-11-17T03:14:53,320 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f8f834df9f47d3bcdf99baa8e445a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=882, earliestPutTs=1731813292305 2024-11-17T03:14:53,323 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/A of b29e9d27b06f8df48f1da9682bc65b77 into 8756026b3dcf478fad64eacafd71ae14(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:53,323 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:53,324 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/A, priority=13, startTime=1731813293279; duration=0sec 2024-11-17T03:14:53,324 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:53,324 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:A 2024-11-17T03:14:53,328 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b29e9d27b06f8df48f1da9682bc65b77#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:14:53,329 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/960719f6fd564f06a148f698682337a7 is 50, key is test_row_0/C:col10/1731813292417/Put/seqid=0 2024-11-17T03:14:53,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741941_1117 (size=13459) 2024-11-17T03:14:53,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T03:14:53,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:53,348 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T03:14:53,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:14:53,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:53,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:14:53,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:53,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:14:53,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:14:53,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/38670bdbb10347c1b678cd8bdefc27d8 is 50, key is test_row_0/A:col10/1731813292440/Put/seqid=0 2024-11-17T03:14:53,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741942_1118 (size=12301) 2024-11-17T03:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T03:14:53,538 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T03:14:53,538 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:14:53,539 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T03:14:53,539 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:14:53,543 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,543 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,543 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T03:14:53,543 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:14:53,543 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,544 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,544 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:14:53,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:14:53,544 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:14:53,546 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T03:14:53,546 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:14:53,546 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,546 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:14:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:14:53,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. as already flushing 2024-11-17T03:14:53,590 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T03:14:53,590 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T03:14:53,591 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:14:53,591 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:14:53,591 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,591 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,591 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,591 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:14:53,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:14:53,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:14:53,747 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/960719f6fd564f06a148f698682337a7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/960719f6fd564f06a148f698682337a7 2024-11-17T03:14:53,753 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b29e9d27b06f8df48f1da9682bc65b77/C of b29e9d27b06f8df48f1da9682bc65b77 into 960719f6fd564f06a148f698682337a7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:14:53,753 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:53,753 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., storeName=b29e9d27b06f8df48f1da9682bc65b77/C, priority=13, startTime=1731813293279; duration=0sec 2024-11-17T03:14:53,753 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:14:53,754 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b29e9d27b06f8df48f1da9682bc65b77:C 2024-11-17T03:14:53,758 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=904 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/38670bdbb10347c1b678cd8bdefc27d8 2024-11-17T03:14:53,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/5f1d4e22456f4d8386391eefddef2503 is 50, key is test_row_0/B:col10/1731813292440/Put/seqid=0 2024-11-17T03:14:53,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741943_1119 (size=12301) 2024-11-17T03:14:54,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T03:14:54,176 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=904 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/5f1d4e22456f4d8386391eefddef2503 2024-11-17T03:14:54,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/c4552ddee0db4f57b3219a915511035e is 50, key is test_row_0/C:col10/1731813292440/Put/seqid=0 2024-11-17T03:14:54,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741944_1120 (size=12301) 2024-11-17T03:14:54,352 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T03:14:54,596 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=904 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/c4552ddee0db4f57b3219a915511035e 2024-11-17T03:14:54,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/38670bdbb10347c1b678cd8bdefc27d8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38670bdbb10347c1b678cd8bdefc27d8 2024-11-17T03:14:54,615 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38670bdbb10347c1b678cd8bdefc27d8, entries=150, sequenceid=904, filesize=12.0 K 2024-11-17T03:14:54,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/5f1d4e22456f4d8386391eefddef2503 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5f1d4e22456f4d8386391eefddef2503 2024-11-17T03:14:54,623 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5f1d4e22456f4d8386391eefddef2503, entries=150, sequenceid=904, filesize=12.0 K 2024-11-17T03:14:54,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/c4552ddee0db4f57b3219a915511035e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/c4552ddee0db4f57b3219a915511035e 2024-11-17T03:14:54,631 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/c4552ddee0db4f57b3219a915511035e, entries=150, sequenceid=904, filesize=12.0 K 2024-11-17T03:14:54,632 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for b29e9d27b06f8df48f1da9682bc65b77 in 1284ms, sequenceid=904, compaction requested=false 2024-11-17T03:14:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for b29e9d27b06f8df48f1da9682bc65b77: 2024-11-17T03:14:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:14:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T03:14:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T03:14:54,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T03:14:54,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7440 sec 2024-11-17T03:14:54,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7480 sec 2024-11-17T03:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T03:14:55,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:00,452 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T03:15:00,452 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:00,453 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,453 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:00,501 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T03:15:00,501 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:00,502 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,502 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:00,548 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T03:15:00,548 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:00,548 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,549 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,549 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 274 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 327 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2744 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2727 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1207 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3620 rows 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1215 2024-11-17T03:15:00,549 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3644 rows 2024-11-17T03:15:00,549 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:15:00,550 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:15:00,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:00,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:00,553 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T03:15:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T03:15:00,560 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813300560"}]},"ts":"1731813300560"} 2024-11-17T03:15:00,562 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T03:15:00,563 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T03:15:00,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:15:00,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, UNASSIGN}] 2024-11-17T03:15:00,572 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, UNASSIGN 2024-11-17T03:15:00,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b29e9d27b06f8df48f1da9682bc65b77, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:00,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, UNASSIGN because future has completed 2024-11-17T03:15:00,577 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:15:00,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T03:15:00,735 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:15:00,735 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing b29e9d27b06f8df48f1da9682bc65b77, disabling compactions & flushes 2024-11-17T03:15:00,736 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. after waiting 0 ms 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:15:00,736 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing b29e9d27b06f8df48f1da9682bc65b77 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=A 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=B 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b29e9d27b06f8df48f1da9682bc65b77, store=C 2024-11-17T03:15:00,736 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:00,741 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/1ea428cc83684b67a78bc808b0a144a6 is 50, key is test_row_0/A:col10/1731813293588/Put/seqid=0 2024-11-17T03:15:00,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741945_1121 (size=12301) 2024-11-17T03:15:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T03:15:01,147 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=913 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/1ea428cc83684b67a78bc808b0a144a6 2024-11-17T03:15:01,162 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3d69d236d444d1fa6d76c613976bf21 is 50, key is test_row_0/B:col10/1731813293588/Put/seqid=0 2024-11-17T03:15:01,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741946_1122 (size=12301) 2024-11-17T03:15:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T03:15:01,567 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=913 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3d69d236d444d1fa6d76c613976bf21 2024-11-17T03:15:01,582 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/7a8a8a8d43004e599f34b796844af69d is 50, key is test_row_0/C:col10/1731813293588/Put/seqid=0 2024-11-17T03:15:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741947_1123 (size=12301) 2024-11-17T03:15:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T03:15:02,031 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=913 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/7a8a8a8d43004e599f34b796844af69d 2024-11-17T03:15:02,039 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/A/1ea428cc83684b67a78bc808b0a144a6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/1ea428cc83684b67a78bc808b0a144a6 2024-11-17T03:15:02,046 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/1ea428cc83684b67a78bc808b0a144a6, entries=150, sequenceid=913, filesize=12.0 K 2024-11-17T03:15:02,047 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/B/b3d69d236d444d1fa6d76c613976bf21 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3d69d236d444d1fa6d76c613976bf21 2024-11-17T03:15:02,053 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3d69d236d444d1fa6d76c613976bf21, entries=150, sequenceid=913, filesize=12.0 K 2024-11-17T03:15:02,054 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/.tmp/C/7a8a8a8d43004e599f34b796844af69d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/7a8a8a8d43004e599f34b796844af69d 2024-11-17T03:15:02,058 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/7a8a8a8d43004e599f34b796844af69d, entries=150, sequenceid=913, filesize=12.0 K 2024-11-17T03:15:02,059 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b29e9d27b06f8df48f1da9682bc65b77 in 1323ms, sequenceid=913, compaction requested=true 2024-11-17T03:15:02,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f6de317de0db41279fe4a50e7d47a621, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38092d9db1364b5d885ddfa5ea6d8a9b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/48131e4b74384361ab419cd2a81603ae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/74898e9355514095ae49046364821894, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d38b7dac745542038cdfc48f0e9a0ddc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4c7ead0dbf4e4ad18bb6e6be14bcb313, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/494dcc6c31e74ca1b64b08d1dfae869c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/890e3e8387ee4277b6c7658155693a17, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/67f979c9ed274d74b402cd1de57c736a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a8f3c19ccd49407a999a56ac9802e18e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f36d5d399ef041dd9bae4d2c6b04999f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/9071396afcc04ab9bd35d2f34f436fe3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/6f9d6f1f50c546dcb80fcf8178a29850, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/83de486b352d4395a4e4bea743521e4f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a9f8b8c77391403cb1f3ce5213934df1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7d24f3f01a1142c5a0ef6110e9e03ce4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7241231d7e28454f91e93d778be67f79, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b72572d430584a9ebea89161a51f7817, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/69ded74da5d945ed9efdbcbd72a0418c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d44c2d9610f043afb756e947fe040149, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d36c32af0cd34b568960ccd995d5386d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ec51e5510ac84c77a0c25b118ae54a18, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/566fbbf2e2d542b6a344b4ab73a222a7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/e877b414dbcd45afbbcd4e1bcc9aefd1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/45eba99ab8d04c08b30ec845d6ffc0db, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/c2a3f74a07ba487880fff26ac4121f91, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ecbf8d81f1df40d4a6f80aa3002d792a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4299640812bf4aac86070167da1b9e12, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/050774fdb7734872b67b96637ac4f243, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a80790f70ea44838915f888ec9e98bde, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/3e3520e7c9d8490f89ca93ac6fe13d97, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/aba477694cc54dff9bbd818342e1a2cc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8e9d2d11985e4926a4098fdd8d7f8d1c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b996019ed3da4e00bcebd47f978fa709] to archive 2024-11-17T03:15:02,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:02,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f6de317de0db41279fe4a50e7d47a621 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f6de317de0db41279fe4a50e7d47a621 2024-11-17T03:15:02,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38092d9db1364b5d885ddfa5ea6d8a9b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38092d9db1364b5d885ddfa5ea6d8a9b 2024-11-17T03:15:02,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/48131e4b74384361ab419cd2a81603ae to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/48131e4b74384361ab419cd2a81603ae 2024-11-17T03:15:02,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/74898e9355514095ae49046364821894 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/74898e9355514095ae49046364821894 2024-11-17T03:15:02,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d38b7dac745542038cdfc48f0e9a0ddc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d38b7dac745542038cdfc48f0e9a0ddc 2024-11-17T03:15:02,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4c7ead0dbf4e4ad18bb6e6be14bcb313 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4c7ead0dbf4e4ad18bb6e6be14bcb313 2024-11-17T03:15:02,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/494dcc6c31e74ca1b64b08d1dfae869c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/494dcc6c31e74ca1b64b08d1dfae869c 2024-11-17T03:15:02,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/890e3e8387ee4277b6c7658155693a17 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/890e3e8387ee4277b6c7658155693a17 2024-11-17T03:15:02,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/67f979c9ed274d74b402cd1de57c736a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/67f979c9ed274d74b402cd1de57c736a 2024-11-17T03:15:02,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a8f3c19ccd49407a999a56ac9802e18e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a8f3c19ccd49407a999a56ac9802e18e 2024-11-17T03:15:02,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f36d5d399ef041dd9bae4d2c6b04999f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/f36d5d399ef041dd9bae4d2c6b04999f 2024-11-17T03:15:02,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/9071396afcc04ab9bd35d2f34f436fe3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/9071396afcc04ab9bd35d2f34f436fe3 2024-11-17T03:15:02,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/6f9d6f1f50c546dcb80fcf8178a29850 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/6f9d6f1f50c546dcb80fcf8178a29850 2024-11-17T03:15:02,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/83de486b352d4395a4e4bea743521e4f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/83de486b352d4395a4e4bea743521e4f 2024-11-17T03:15:02,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a9f8b8c77391403cb1f3ce5213934df1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a9f8b8c77391403cb1f3ce5213934df1 2024-11-17T03:15:02,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7d24f3f01a1142c5a0ef6110e9e03ce4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7d24f3f01a1142c5a0ef6110e9e03ce4 2024-11-17T03:15:02,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7241231d7e28454f91e93d778be67f79 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/7241231d7e28454f91e93d778be67f79 2024-11-17T03:15:02,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b72572d430584a9ebea89161a51f7817 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b72572d430584a9ebea89161a51f7817 2024-11-17T03:15:02,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/69ded74da5d945ed9efdbcbd72a0418c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/69ded74da5d945ed9efdbcbd72a0418c 2024-11-17T03:15:02,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d44c2d9610f043afb756e947fe040149 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d44c2d9610f043afb756e947fe040149 2024-11-17T03:15:02,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d36c32af0cd34b568960ccd995d5386d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/d36c32af0cd34b568960ccd995d5386d 2024-11-17T03:15:02,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ec51e5510ac84c77a0c25b118ae54a18 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ec51e5510ac84c77a0c25b118ae54a18 2024-11-17T03:15:02,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/566fbbf2e2d542b6a344b4ab73a222a7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/566fbbf2e2d542b6a344b4ab73a222a7 2024-11-17T03:15:02,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/e877b414dbcd45afbbcd4e1bcc9aefd1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/e877b414dbcd45afbbcd4e1bcc9aefd1 2024-11-17T03:15:02,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/45eba99ab8d04c08b30ec845d6ffc0db to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/45eba99ab8d04c08b30ec845d6ffc0db 2024-11-17T03:15:02,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/c2a3f74a07ba487880fff26ac4121f91 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/c2a3f74a07ba487880fff26ac4121f91 2024-11-17T03:15:02,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ecbf8d81f1df40d4a6f80aa3002d792a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/ecbf8d81f1df40d4a6f80aa3002d792a 2024-11-17T03:15:02,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4299640812bf4aac86070167da1b9e12 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/4299640812bf4aac86070167da1b9e12 2024-11-17T03:15:02,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/050774fdb7734872b67b96637ac4f243 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/050774fdb7734872b67b96637ac4f243 2024-11-17T03:15:02,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a80790f70ea44838915f888ec9e98bde to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/a80790f70ea44838915f888ec9e98bde 2024-11-17T03:15:02,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/3e3520e7c9d8490f89ca93ac6fe13d97 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/3e3520e7c9d8490f89ca93ac6fe13d97 2024-11-17T03:15:02,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/aba477694cc54dff9bbd818342e1a2cc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/aba477694cc54dff9bbd818342e1a2cc 2024-11-17T03:15:02,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8e9d2d11985e4926a4098fdd8d7f8d1c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8e9d2d11985e4926a4098fdd8d7f8d1c 2024-11-17T03:15:02,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b996019ed3da4e00bcebd47f978fa709 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/b996019ed3da4e00bcebd47f978fa709 2024-11-17T03:15:02,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/096eb8198cfe4610a81ab398c589f594, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc04b6b0c8794be79e154a839a898ade, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/90e611bb295f41d58c2ecac158f6db5d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b7a6165c02e34ee2bec1cd3a96dc9b3c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/da76a10ea4db4fb1a64f14d396e17d21, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6989e046799a4044bedb7fb8d68b603b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3bf311228da44419d58cd3a54607624, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/7807580cd8a343d3916c08f218755fa0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/d8132433858f4470801adc6c94f29010, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/91adce760a484c6482ad92bebefd9021, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6e57b6e847874055a90a764ba778da82, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a5ce08fdebe44b57b6c737a6211c4b76, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a409dfcc3f8342c6890952e8a5fa9d38, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/697397bd974b4140b53418e666a08906, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc098513ec4c44089eb234786b86c18b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/dce1e2701ebf469899116b6c1efd0e9c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cc4aed477ea945f4af9fe25f047cada4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ed74b22f8fc545a6a676892834827be9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b72f311646b842c0b46603bd418ead47, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/01697bf16b3640419586e24856c36af3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/92f5b93393004a278c38076d80b07000, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/55a6489cc9e6412b96eb273d372ec4e7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/8e771394e711461ba59f2e2c4b6b4893, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbbf1164330e4581b893b202c8116534, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/73ede981602544569217ed6e0d167e9c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a2bf8e94b2904604924256d373472093, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5e80dd8bcb6641ebbf0a7562cb47429e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3a7abda1c8f4745b0831e0dc56a1514, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/979ad4df2de742bea0929f346c87403c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bdddb4d1d5614456b95a673797de9aa9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cda13c2bdbf049a7a3ab2e9bbc991ee5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e678f006be3e4b31802be2e252c4f072] to archive 2024-11-17T03:15:02,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:02,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/096eb8198cfe4610a81ab398c589f594 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/096eb8198cfe4610a81ab398c589f594 2024-11-17T03:15:02,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc04b6b0c8794be79e154a839a898ade to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc04b6b0c8794be79e154a839a898ade 2024-11-17T03:15:02,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/90e611bb295f41d58c2ecac158f6db5d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/90e611bb295f41d58c2ecac158f6db5d 2024-11-17T03:15:02,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b7a6165c02e34ee2bec1cd3a96dc9b3c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b7a6165c02e34ee2bec1cd3a96dc9b3c 2024-11-17T03:15:02,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/da76a10ea4db4fb1a64f14d396e17d21 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/da76a10ea4db4fb1a64f14d396e17d21 2024-11-17T03:15:02,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6989e046799a4044bedb7fb8d68b603b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6989e046799a4044bedb7fb8d68b603b 2024-11-17T03:15:02,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3bf311228da44419d58cd3a54607624 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3bf311228da44419d58cd3a54607624 2024-11-17T03:15:02,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/7807580cd8a343d3916c08f218755fa0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/7807580cd8a343d3916c08f218755fa0 2024-11-17T03:15:02,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/d8132433858f4470801adc6c94f29010 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/d8132433858f4470801adc6c94f29010 2024-11-17T03:15:02,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/91adce760a484c6482ad92bebefd9021 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/91adce760a484c6482ad92bebefd9021 2024-11-17T03:15:02,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6e57b6e847874055a90a764ba778da82 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/6e57b6e847874055a90a764ba778da82 2024-11-17T03:15:02,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbfaa6ffbdcb4f0bbb4a3e39ccf5363f 2024-11-17T03:15:02,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a5ce08fdebe44b57b6c737a6211c4b76 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a5ce08fdebe44b57b6c737a6211c4b76 2024-11-17T03:15:02,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a409dfcc3f8342c6890952e8a5fa9d38 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a409dfcc3f8342c6890952e8a5fa9d38 2024-11-17T03:15:02,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/697397bd974b4140b53418e666a08906 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/697397bd974b4140b53418e666a08906 2024-11-17T03:15:02,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc098513ec4c44089eb234786b86c18b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bc098513ec4c44089eb234786b86c18b 2024-11-17T03:15:02,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/dce1e2701ebf469899116b6c1efd0e9c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/dce1e2701ebf469899116b6c1efd0e9c 2024-11-17T03:15:02,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cc4aed477ea945f4af9fe25f047cada4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cc4aed477ea945f4af9fe25f047cada4 2024-11-17T03:15:02,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ed74b22f8fc545a6a676892834827be9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ed74b22f8fc545a6a676892834827be9 2024-11-17T03:15:02,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b72f311646b842c0b46603bd418ead47 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b72f311646b842c0b46603bd418ead47 2024-11-17T03:15:02,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/01697bf16b3640419586e24856c36af3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/01697bf16b3640419586e24856c36af3 2024-11-17T03:15:02,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/92f5b93393004a278c38076d80b07000 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/92f5b93393004a278c38076d80b07000 2024-11-17T03:15:02,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/55a6489cc9e6412b96eb273d372ec4e7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/55a6489cc9e6412b96eb273d372ec4e7 2024-11-17T03:15:02,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/8e771394e711461ba59f2e2c4b6b4893 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/8e771394e711461ba59f2e2c4b6b4893 2024-11-17T03:15:02,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbbf1164330e4581b893b202c8116534 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/fbbf1164330e4581b893b202c8116534 2024-11-17T03:15:02,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/ae61f17cf85c4bc5b7f4ab5fd98d3f9d 2024-11-17T03:15:02,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/73ede981602544569217ed6e0d167e9c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/73ede981602544569217ed6e0d167e9c 2024-11-17T03:15:02,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a2bf8e94b2904604924256d373472093 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/a2bf8e94b2904604924256d373472093 2024-11-17T03:15:02,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5e80dd8bcb6641ebbf0a7562cb47429e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5e80dd8bcb6641ebbf0a7562cb47429e 2024-11-17T03:15:02,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3a7abda1c8f4745b0831e0dc56a1514 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3a7abda1c8f4745b0831e0dc56a1514 2024-11-17T03:15:02,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/979ad4df2de742bea0929f346c87403c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/979ad4df2de742bea0929f346c87403c 2024-11-17T03:15:02,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bdddb4d1d5614456b95a673797de9aa9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/bdddb4d1d5614456b95a673797de9aa9 2024-11-17T03:15:02,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cda13c2bdbf049a7a3ab2e9bbc991ee5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/cda13c2bdbf049a7a3ab2e9bbc991ee5 2024-11-17T03:15:02,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e678f006be3e4b31802be2e252c4f072 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e678f006be3e4b31802be2e252c4f072 2024-11-17T03:15:02,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1e9e667b874b4b278fb6259d81ee73a9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6d718a8685814a6eb480b761cbf86d96, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/72c778c9fba84d75b3d5aa4d0d975666, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4db4777226545af824bc6a9e01d29f4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ebe040ff05ae4e81a44dfe4030703cad, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ff454110596f40fd8bebd486ad3c4b20, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e0cb3416c5af4bc8b6b035bcc9c7f093, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/31a460f84a714d12bd1530b0c284a247, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ddb112005c904ca59ecdcb51be498831, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ac54f270a4ec41f1bb5e0de114407e05, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/084103abfde045b5a75eee08c4ee86a5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/437f5b7c20054ba0a7b52fdf937043aa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/deb56902a557447d9d05def0dae1ba32, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e20847b1a477465a8a58b64464c00622, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/3c9a59823de148eb9957b5f5ff4a22e3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/8482a62d2c8f425a9e6381bf8ee8aaf0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/381179b01b7742fd95d8c1e228a5081c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a417d764e4a244238525babbfb8eb8f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ce6a7c7fdd24430898a3048e1252b045, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/cd7761d15e85413cbe8469e253e2ec81, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a4d996227e884d0897e1ebe16b36e1f0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4c88ab601f34c6c94ab05baa9fa1b03, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/4dcf8587af71410baadbfa7c26a553cd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/f129310823ed4ffb809fe6343996cbc3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1c43705a7ab54a31bc64d51b92b45ebd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/81b2054f522c4cf283ce51f7de7f023d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/63940ede9b464001bfb4f0d44ca79bfc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6fb233febcee475bb44a9a25a3c4fa45, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/10359b7e7cf84a01baa298c8a110e239, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/af66a0cb6b434b8db0a0925dd993d505, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/99c7125866e4455d887c241c038d7194, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ae95d29798d946fe8726aa794ba53ad1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/54bd9b8944904eaaa224303ccff5fa25, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/23f8f834df9f47d3bcdf99baa8e445a2] to archive 2024-11-17T03:15:02,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:02,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1e9e667b874b4b278fb6259d81ee73a9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1e9e667b874b4b278fb6259d81ee73a9 2024-11-17T03:15:02,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6d718a8685814a6eb480b761cbf86d96 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6d718a8685814a6eb480b761cbf86d96 2024-11-17T03:15:02,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/72c778c9fba84d75b3d5aa4d0d975666 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/72c778c9fba84d75b3d5aa4d0d975666 2024-11-17T03:15:02,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4db4777226545af824bc6a9e01d29f4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4db4777226545af824bc6a9e01d29f4 2024-11-17T03:15:02,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ebe040ff05ae4e81a44dfe4030703cad to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ebe040ff05ae4e81a44dfe4030703cad 2024-11-17T03:15:02,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ff454110596f40fd8bebd486ad3c4b20 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ff454110596f40fd8bebd486ad3c4b20 2024-11-17T03:15:02,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e0cb3416c5af4bc8b6b035bcc9c7f093 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e0cb3416c5af4bc8b6b035bcc9c7f093 2024-11-17T03:15:02,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/31a460f84a714d12bd1530b0c284a247 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/31a460f84a714d12bd1530b0c284a247 2024-11-17T03:15:02,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ddb112005c904ca59ecdcb51be498831 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ddb112005c904ca59ecdcb51be498831 2024-11-17T03:15:02,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ac54f270a4ec41f1bb5e0de114407e05 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ac54f270a4ec41f1bb5e0de114407e05 2024-11-17T03:15:02,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/084103abfde045b5a75eee08c4ee86a5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/084103abfde045b5a75eee08c4ee86a5 2024-11-17T03:15:02,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/437f5b7c20054ba0a7b52fdf937043aa to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/437f5b7c20054ba0a7b52fdf937043aa 2024-11-17T03:15:02,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/deb56902a557447d9d05def0dae1ba32 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/deb56902a557447d9d05def0dae1ba32 2024-11-17T03:15:02,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e20847b1a477465a8a58b64464c00622 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e20847b1a477465a8a58b64464c00622 2024-11-17T03:15:02,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/3c9a59823de148eb9957b5f5ff4a22e3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/3c9a59823de148eb9957b5f5ff4a22e3 2024-11-17T03:15:02,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/8482a62d2c8f425a9e6381bf8ee8aaf0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/8482a62d2c8f425a9e6381bf8ee8aaf0 2024-11-17T03:15:02,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/381179b01b7742fd95d8c1e228a5081c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/381179b01b7742fd95d8c1e228a5081c 2024-11-17T03:15:02,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a417d764e4a244238525babbfb8eb8f1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a417d764e4a244238525babbfb8eb8f1 2024-11-17T03:15:02,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ce6a7c7fdd24430898a3048e1252b045 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ce6a7c7fdd24430898a3048e1252b045 2024-11-17T03:15:02,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/cd7761d15e85413cbe8469e253e2ec81 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/cd7761d15e85413cbe8469e253e2ec81 2024-11-17T03:15:02,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a4d996227e884d0897e1ebe16b36e1f0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/a4d996227e884d0897e1ebe16b36e1f0 2024-11-17T03:15:02,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4c88ab601f34c6c94ab05baa9fa1b03 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/e4c88ab601f34c6c94ab05baa9fa1b03 2024-11-17T03:15:02,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/4dcf8587af71410baadbfa7c26a553cd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/4dcf8587af71410baadbfa7c26a553cd 2024-11-17T03:15:02,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/f129310823ed4ffb809fe6343996cbc3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/f129310823ed4ffb809fe6343996cbc3 2024-11-17T03:15:02,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1c43705a7ab54a31bc64d51b92b45ebd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/1c43705a7ab54a31bc64d51b92b45ebd 2024-11-17T03:15:02,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/81b2054f522c4cf283ce51f7de7f023d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/81b2054f522c4cf283ce51f7de7f023d 2024-11-17T03:15:02,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/63940ede9b464001bfb4f0d44ca79bfc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/63940ede9b464001bfb4f0d44ca79bfc 2024-11-17T03:15:02,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6fb233febcee475bb44a9a25a3c4fa45 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/6fb233febcee475bb44a9a25a3c4fa45 2024-11-17T03:15:02,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/10359b7e7cf84a01baa298c8a110e239 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/10359b7e7cf84a01baa298c8a110e239 2024-11-17T03:15:02,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/af66a0cb6b434b8db0a0925dd993d505 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/af66a0cb6b434b8db0a0925dd993d505 2024-11-17T03:15:02,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/99c7125866e4455d887c241c038d7194 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/99c7125866e4455d887c241c038d7194 2024-11-17T03:15:02,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ae95d29798d946fe8726aa794ba53ad1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/ae95d29798d946fe8726aa794ba53ad1 2024-11-17T03:15:02,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/54bd9b8944904eaaa224303ccff5fa25 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/54bd9b8944904eaaa224303ccff5fa25 2024-11-17T03:15:02,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/23f8f834df9f47d3bcdf99baa8e445a2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/23f8f834df9f47d3bcdf99baa8e445a2 2024-11-17T03:15:02,195 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/recovered.edits/916.seqid, newMaxSeqId=916, maxSeqId=1 2024-11-17T03:15:02,195 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. 2024-11-17T03:15:02,195 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for b29e9d27b06f8df48f1da9682bc65b77: Waiting for close lock at 1731813300736Running coprocessor pre-close hooks at 1731813300736Disabling compacts and flushes for region at 1731813300736Disabling writes for close at 1731813300736Obtaining lock to block concurrent updates at 1731813300736Preparing flush snapshotting stores in b29e9d27b06f8df48f1da9682bc65b77 at 1731813300736Finished memstore snapshotting TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731813300736Flushing stores of TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77. at 1731813300737 (+1 ms)Flushing b29e9d27b06f8df48f1da9682bc65b77/A: creating writer at 1731813300737Flushing b29e9d27b06f8df48f1da9682bc65b77/A: appending metadata at 1731813300740 (+3 ms)Flushing b29e9d27b06f8df48f1da9682bc65b77/A: closing flushed file at 1731813300740Flushing b29e9d27b06f8df48f1da9682bc65b77/B: creating writer at 1731813301158 (+418 ms)Flushing b29e9d27b06f8df48f1da9682bc65b77/B: appending metadata at 1731813301161 (+3 ms)Flushing b29e9d27b06f8df48f1da9682bc65b77/B: closing flushed file at 1731813301161Flushing b29e9d27b06f8df48f1da9682bc65b77/C: creating writer at 1731813301576 (+415 ms)Flushing b29e9d27b06f8df48f1da9682bc65b77/C: appending metadata at 1731813301581 (+5 ms)Flushing b29e9d27b06f8df48f1da9682bc65b77/C: closing flushed file at 1731813301581Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4625bb8a: reopening flushed file at 1731813302038 (+457 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2266757d: reopening flushed file at 1731813302047 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3de17a72: reopening flushed file at 1731813302053 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b29e9d27b06f8df48f1da9682bc65b77 in 1323ms, sequenceid=913, compaction requested=true at 1731813302059 (+6 ms)Writing region close event to WAL at 1731813302191 (+132 ms)Running coprocessor post-close hooks at 1731813302195 (+4 ms)Closed at 1731813302195 2024-11-17T03:15:02,198 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:15:02,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b29e9d27b06f8df48f1da9682bc65b77, regionState=CLOSED 2024-11-17T03:15:02,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:02,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T03:15:02,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure b29e9d27b06f8df48f1da9682bc65b77, server=695c90f4bb18,38753,1731813266381 in 1.6240 sec 2024-11-17T03:15:02,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-17T03:15:02,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b29e9d27b06f8df48f1da9682bc65b77, UNASSIGN in 1.6320 sec 2024-11-17T03:15:02,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T03:15:02,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6410 sec 2024-11-17T03:15:02,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813302207"}]},"ts":"1731813302207"} 2024-11-17T03:15:02,208 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T03:15:02,208 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T03:15:02,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6550 sec 2024-11-17T03:15:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T03:15:02,710 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:02,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T03:15:02,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,720 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T03:15:02,721 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,726 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:15:02,730 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/recovered.edits] 2024-11-17T03:15:02,734 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/1ea428cc83684b67a78bc808b0a144a6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/1ea428cc83684b67a78bc808b0a144a6 2024-11-17T03:15:02,735 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38670bdbb10347c1b678cd8bdefc27d8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/38670bdbb10347c1b678cd8bdefc27d8 2024-11-17T03:15:02,737 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8756026b3dcf478fad64eacafd71ae14 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/A/8756026b3dcf478fad64eacafd71ae14 2024-11-17T03:15:02,740 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5f1d4e22456f4d8386391eefddef2503 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/5f1d4e22456f4d8386391eefddef2503 2024-11-17T03:15:02,741 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3d69d236d444d1fa6d76c613976bf21 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/b3d69d236d444d1fa6d76c613976bf21 2024-11-17T03:15:02,742 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e672a5be031d4dd69fafa9e432b7fd4a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/B/e672a5be031d4dd69fafa9e432b7fd4a 2024-11-17T03:15:02,744 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/7a8a8a8d43004e599f34b796844af69d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/7a8a8a8d43004e599f34b796844af69d 2024-11-17T03:15:02,745 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/960719f6fd564f06a148f698682337a7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/960719f6fd564f06a148f698682337a7 2024-11-17T03:15:02,746 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/c4552ddee0db4f57b3219a915511035e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/C/c4552ddee0db4f57b3219a915511035e 2024-11-17T03:15:02,749 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/recovered.edits/916.seqid to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77/recovered.edits/916.seqid 2024-11-17T03:15:02,749 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/b29e9d27b06f8df48f1da9682bc65b77 2024-11-17T03:15:02,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T03:15:02,754 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T03:15:02,762 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T03:15:02,795 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T03:15:02,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,797 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T03:15:02,798 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731813302797"}]},"ts":"9223372036854775807"} 2024-11-17T03:15:02,802 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T03:15:02,802 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b29e9d27b06f8df48f1da9682bc65b77, NAME => 'TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T03:15:02,802 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T03:15:02,803 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731813302802"}]},"ts":"9223372036854775807"} 2024-11-17T03:15:02,806 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T03:15:02,807 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-17T03:15:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T03:15:02,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T03:15:02,829 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:02,839 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;695c90f4bb18:38753-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-131647061_22 at /127.0.0.1:36022 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 342) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1271 (was 1941) 2024-11-17T03:15:02,847 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=1271 2024-11-17T03:15:02,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:15:02,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:15:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:02,851 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T03:15:02,851 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:02,851 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-17T03:15:02,852 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T03:15:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T03:15:02,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741948_1124 (size=1061) 2024-11-17T03:15:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T03:15:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T03:15:03,263 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:15:03,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741949_1125 (size=53) 2024-11-17T03:15:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T03:15:03,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:03,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 42b167b16e2c28db0ea066b1d94c2c1f, disabling compactions & flushes 2024-11-17T03:15:03,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:03,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:03,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. after waiting 0 ms 2024-11-17T03:15:03,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:03,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:03,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42b167b16e2c28db0ea066b1d94c2c1f: Waiting for close lock at 1731813303674Disabling compacts and flushes for region at 1731813303674Disabling writes for close at 1731813303675 (+1 ms)Writing region close event to WAL at 1731813303675Closed at 1731813303675 2024-11-17T03:15:03,676 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T03:15:03,677 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731813303676"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731813303676"}]},"ts":"1731813303676"} 2024-11-17T03:15:03,679 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T03:15:03,680 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T03:15:03,680 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813303680"}]},"ts":"1731813303680"} 2024-11-17T03:15:03,682 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T03:15:03,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, ASSIGN}] 2024-11-17T03:15:03,684 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, ASSIGN 2024-11-17T03:15:03,685 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, ASSIGN; state=OFFLINE, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=false 2024-11-17T03:15:03,837 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:03,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, ASSIGN because future has completed 2024-11-17T03:15:03,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T03:15:03,998 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:03,998 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:15:03,998 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:03,998 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:03,999 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:03,999 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,000 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,001 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:04,002 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b167b16e2c28db0ea066b1d94c2c1f columnFamilyName A 2024-11-17T03:15:04,003 DEBUG [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:04,003 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(327): Store=42b167b16e2c28db0ea066b1d94c2c1f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:04,003 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,005 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:04,005 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b167b16e2c28db0ea066b1d94c2c1f columnFamilyName B 2024-11-17T03:15:04,005 DEBUG [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:04,006 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(327): Store=42b167b16e2c28db0ea066b1d94c2c1f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:04,006 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,007 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:04,007 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b167b16e2c28db0ea066b1d94c2c1f columnFamilyName C 2024-11-17T03:15:04,007 DEBUG [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:04,008 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(327): Store=42b167b16e2c28db0ea066b1d94c2c1f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:04,008 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:04,008 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,008 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,009 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,010 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,010 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,010 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:15:04,011 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,013 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:15:04,014 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 42b167b16e2c28db0ea066b1d94c2c1f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69130180, jitterRate=0.03011995553970337}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:15:04,014 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:04,015 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 42b167b16e2c28db0ea066b1d94c2c1f: Running coprocessor pre-open hook at 1731813303999Writing region info on filesystem at 1731813303999Initializing all the Stores at 1731813303999Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813304000 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813304000Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813304000Cleaning up temporary data from old regions at 1731813304010 (+10 ms)Running coprocessor post-open hooks at 1731813304014 (+4 ms)Region opened successfully at 1731813304015 (+1 ms) 2024-11-17T03:15:04,015 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., pid=34, masterSystemTime=1731813303995 2024-11-17T03:15:04,017 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:04,017 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:04,018 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:04,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:04,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-17T03:15:04,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 in 178 msec 2024-11-17T03:15:04,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-17T03:15:04,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, ASSIGN in 341 msec 2024-11-17T03:15:04,025 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T03:15:04,026 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813304025"}]},"ts":"1731813304025"} 2024-11-17T03:15:04,027 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T03:15:04,028 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T03:15:04,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-17T03:15:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-17T03:15:04,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:04,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:04,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:04,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:05,000 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f17889c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:05,002 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329c9ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,003 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:05,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:05,006 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eb25f6a 2024-11-17T03:15:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:15:05,008 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:15:05,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:15:05,013 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:15:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:05,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741950_1126 (size=1097) 2024-11-17T03:15:05,433 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T03:15:05,434 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T03:15:05,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:15:05,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, REOPEN/MOVE}] 2024-11-17T03:15:05,446 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, REOPEN/MOVE 2024-11-17T03:15:05,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:05,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, REOPEN/MOVE because future has completed 2024-11-17T03:15:05,448 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:15:05,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:05,602 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,602 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:15:05,602 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 42b167b16e2c28db0ea066b1d94c2c1f, disabling compactions & flushes 2024-11-17T03:15:05,602 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,602 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,602 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. after waiting 0 ms 2024-11-17T03:15:05,602 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,610 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T03:15:05,611 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,611 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 42b167b16e2c28db0ea066b1d94c2c1f: Waiting for close lock at 1731813305602Running coprocessor pre-close hooks at 1731813305602Disabling compacts and flushes for region at 1731813305602Disabling writes for close at 1731813305602Writing region close event to WAL at 1731813305603 (+1 ms)Running coprocessor post-close hooks at 1731813305611 (+8 ms)Closed at 1731813305611 2024-11-17T03:15:05,612 WARN [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 42b167b16e2c28db0ea066b1d94c2c1f to self. 2024-11-17T03:15:05,616 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,617 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=CLOSED 2024-11-17T03:15:05,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:05,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T03:15:05,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 in 172 msec 2024-11-17T03:15:05,622 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, REOPEN/MOVE; state=CLOSED, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=true 2024-11-17T03:15:05,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T03:15:05,773 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:05,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, REOPEN/MOVE because future has completed 2024-11-17T03:15:05,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:05,940 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,940 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:15:05,941 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,941 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:05,941 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,941 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,945 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,946 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:05,951 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b167b16e2c28db0ea066b1d94c2c1f columnFamilyName A 2024-11-17T03:15:05,953 DEBUG [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:05,953 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(327): Store=42b167b16e2c28db0ea066b1d94c2c1f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:05,953 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,954 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:05,954 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b167b16e2c28db0ea066b1d94c2c1f columnFamilyName B 2024-11-17T03:15:05,954 DEBUG [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:05,955 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(327): Store=42b167b16e2c28db0ea066b1d94c2c1f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:05,955 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,955 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:05,956 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42b167b16e2c28db0ea066b1d94c2c1f columnFamilyName C 2024-11-17T03:15:05,956 DEBUG [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:05,956 INFO [StoreOpener-42b167b16e2c28db0ea066b1d94c2c1f-1 {}] regionserver.HStore(327): Store=42b167b16e2c28db0ea066b1d94c2c1f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:05,956 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,956 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,957 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,958 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,959 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,959 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,960 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:15:05,961 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,962 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 42b167b16e2c28db0ea066b1d94c2c1f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71062573, jitterRate=0.05891485512256622}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:15:05,962 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:05,963 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 42b167b16e2c28db0ea066b1d94c2c1f: Running coprocessor pre-open hook at 1731813305942Writing region info on filesystem at 1731813305942Initializing all the Stores at 1731813305943 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813305943Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813305943Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813305943Cleaning up temporary data from old regions at 1731813305959 (+16 ms)Running coprocessor post-open hooks at 1731813305962 (+3 ms)Region opened successfully at 1731813305963 (+1 ms) 2024-11-17T03:15:05,964 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., pid=39, masterSystemTime=1731813305932 2024-11-17T03:15:05,966 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,966 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:05,966 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=OPEN, openSeqNum=5, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:05,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:05,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-17T03:15:05,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 in 192 msec 2024-11-17T03:15:05,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-17T03:15:05,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, REOPEN/MOVE in 525 msec 2024-11-17T03:15:05,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T03:15:05,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-17T03:15:05,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-17T03:15:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-17T03:15:05,982 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:05,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,982 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:05,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:05,983 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:05,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8216eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:05,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:05,985 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e16802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:05,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:05,988 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:05,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c4cbc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:05,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f1a5dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:05,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:05,994 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b62172d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:05,996 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24dc22b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:06,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:06,003 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18971000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:06,005 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebe3853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:06,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:06,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f613ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:06,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:06,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2a598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f1b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:06,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:06,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7543337a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:06,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f557cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62885e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:06,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:06,016 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d8325f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:06,018 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29db9dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138dc9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:06,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:06,021 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f5ff67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:06,022 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28233673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c264dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:06,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:06,024 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14fc1887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:06,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6df162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:06,028 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,028 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,028 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,028 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,029 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,029 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,030 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,030 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,030 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:06,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:06,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,033 DEBUG [Thread-640 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:06,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-17T03:15:06,035 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,036 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:06,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:06,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,041 DEBUG [Thread-640 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:06,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:06,047 DEBUG [Thread-640 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:15:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:06,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:15:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:06,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c78fdced7d3046f5aa855ed257c97f87_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813306051/Put/seqid=0 2024-11-17T03:15:06,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21014, totalHeapSize=47976, min timestamp=1731813306083, max timestamp=1731813306113Num uniques 859; 2024-11-17T03:15:06,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21014, totalHeapSize=47976, min timestamp=1731813306083, max timestamp=1731813306113Num uniques 859; 2024-11-17T03:15:06,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=21014, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-17T03:15:06,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=21014, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-17T03:15:06,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731813306083, max timestamp=1731813306113Num uniques 859; 2024-11-17T03:15:06,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-17T03:15:06,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741951_1127 (size=12154) 2024-11-17T03:15:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813366134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813366136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813366138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813366140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:06,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813366142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813366247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813366247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813366247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813366247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813366257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:06,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813366457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813366458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813366458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813366458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813366467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:06,538 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c78fdced7d3046f5aa855ed257c97f87_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c78fdced7d3046f5aa855ed257c97f87_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:06,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/fc5c58991d4243f9942584d0e3c6efa1, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:06,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/fc5c58991d4243f9942584d0e3c6efa1 is 175, key is test_row_0/A:col10/1731813306051/Put/seqid=0 2024-11-17T03:15:06,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741952_1128 (size=30955) 2024-11-17T03:15:06,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:06,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:06,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813366777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813366777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813366778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813366778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813366778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:06,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:06,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:06,973 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/fc5c58991d4243f9942584d0e3c6efa1 2024-11-17T03:15:07,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/d3e11553da0a40d0b1799e8cfdf06016 is 50, key is test_row_0/B:col10/1731813306051/Put/seqid=0 2024-11-17T03:15:07,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741953_1129 (size=12001) 2024-11-17T03:15:07,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:07,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:07,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:07,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:07,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813367287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813367287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813367287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813367288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:07,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813367297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:07,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/d3e11553da0a40d0b1799e8cfdf06016 2024-11-17T03:15:07,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/0b1b3542c43d40d1a38df159fe44706a is 50, key is test_row_0/C:col10/1731813306051/Put/seqid=0 2024-11-17T03:15:07,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741954_1130 (size=12001) 2024-11-17T03:15:07,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:07,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:07,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:07,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,734 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T03:15:07,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:07,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:07,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/0b1b3542c43d40d1a38df159fe44706a 2024-11-17T03:15:07,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/fc5c58991d4243f9942584d0e3c6efa1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1 2024-11-17T03:15:07,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T03:15:07,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/d3e11553da0a40d0b1799e8cfdf06016 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d3e11553da0a40d0b1799e8cfdf06016 2024-11-17T03:15:07,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d3e11553da0a40d0b1799e8cfdf06016, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T03:15:07,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/0b1b3542c43d40d1a38df159fe44706a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b1b3542c43d40d1a38df159fe44706a 2024-11-17T03:15:07,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b1b3542c43d40d1a38df159fe44706a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T03:15:07,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 42b167b16e2c28db0ea066b1d94c2c1f in 1889ms, sequenceid=16, compaction requested=false 2024-11-17T03:15:07,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:08,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:08,038 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=127.47 KB heapSize=336.94 KB 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:08,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111791d0fba1289a42f1881f7737ca3030be_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813306135/Put/seqid=0 2024-11-17T03:15:08,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741955_1131 (size=12154) 2024-11-17T03:15:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:08,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813368315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813368316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813368324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813368325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813368327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813368437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813368438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813368438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813368438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813368438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:08,465 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111791d0fba1289a42f1881f7737ca3030be_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111791d0fba1289a42f1881f7737ca3030be_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/456ad50e120640df96a9bc1ab8de01c0, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/456ad50e120640df96a9bc1ab8de01c0 is 175, key is test_row_0/A:col10/1731813306135/Put/seqid=0 2024-11-17T03:15:08,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741956_1132 (size=30955) 2024-11-17T03:15:08,471 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/456ad50e120640df96a9bc1ab8de01c0 2024-11-17T03:15:08,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/7883c07793054b08a9b76d99a35b223f is 50, key is test_row_0/B:col10/1731813306135/Put/seqid=0 2024-11-17T03:15:08,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741957_1133 (size=12001) 2024-11-17T03:15:08,499 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/7883c07793054b08a9b76d99a35b223f 2024-11-17T03:15:08,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/6cd5ca95c285465fac6f2404213ed810 is 50, key is test_row_0/C:col10/1731813306135/Put/seqid=0 2024-11-17T03:15:08,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741958_1134 (size=12001) 2024-11-17T03:15:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813368646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813368647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813368647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813368647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813368647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,923 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/6cd5ca95c285465fac6f2404213ed810 2024-11-17T03:15:08,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/456ad50e120640df96a9bc1ab8de01c0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0 2024-11-17T03:15:08,938 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0, entries=150, sequenceid=46, filesize=30.2 K 2024-11-17T03:15:08,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/7883c07793054b08a9b76d99a35b223f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/7883c07793054b08a9b76d99a35b223f 2024-11-17T03:15:08,944 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/7883c07793054b08a9b76d99a35b223f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T03:15:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/6cd5ca95c285465fac6f2404213ed810 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/6cd5ca95c285465fac6f2404213ed810 2024-11-17T03:15:08,950 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/6cd5ca95c285465fac6f2404213ed810, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T03:15:08,951 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~336.89 KB/344976, currentSize=67.09 KB/68700 for 42b167b16e2c28db0ea066b1d94c2c1f in 913ms, sequenceid=46, compaction requested=false 2024-11-17T03:15:08,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:08,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:08,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-17T03:15:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-17T03:15:08,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-17T03:15:08,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9170 sec 2024-11-17T03:15:08,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.9250 sec 2024-11-17T03:15:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:08,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:15:08,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:08,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:08,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:08,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:08,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:08,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:08,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f8ead41ba9ff46ff9afbba28c93d9a26_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:08,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741959_1135 (size=12154) 2024-11-17T03:15:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813368976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813368977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813368979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813368980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:08,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813368981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813369086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813369087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813369097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813369097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813369097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813369297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813369297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813369307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813369307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813369307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:09,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f8ead41ba9ff46ff9afbba28c93d9a26_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f8ead41ba9ff46ff9afbba28c93d9a26_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:09,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/5cf874877c394cabab2edd2a0e91c1dc, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:09,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/5cf874877c394cabab2edd2a0e91c1dc is 175, key is test_row_0/A:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:09,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741960_1136 (size=30955) 2024-11-17T03:15:09,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813369607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813369607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813369617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813369617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813369618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:09,784 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/5cf874877c394cabab2edd2a0e91c1dc 2024-11-17T03:15:09,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/b5832965f8ac48bc967935fdaea1c7d3 is 50, key is test_row_0/B:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:09,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741961_1137 (size=12001) 2024-11-17T03:15:09,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/b5832965f8ac48bc967935fdaea1c7d3 2024-11-17T03:15:09,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/b372e3f3947f425d9baa04f781f9ed64 is 50, key is test_row_0/C:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:09,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741962_1138 (size=12001) 2024-11-17T03:15:10,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813370117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:10,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813370117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:10,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813370129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:10,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813370129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813370129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T03:15:10,198 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:10,199 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T03:15:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T03:15:10,201 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:10,202 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:10,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:10,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/b372e3f3947f425d9baa04f781f9ed64 2024-11-17T03:15:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/5cf874877c394cabab2edd2a0e91c1dc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc 2024-11-17T03:15:10,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc, entries=150, sequenceid=60, filesize=30.2 K 2024-11-17T03:15:10,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/b5832965f8ac48bc967935fdaea1c7d3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b5832965f8ac48bc967935fdaea1c7d3 2024-11-17T03:15:10,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b5832965f8ac48bc967935fdaea1c7d3, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T03:15:10,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/b372e3f3947f425d9baa04f781f9ed64 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/b372e3f3947f425d9baa04f781f9ed64 2024-11-17T03:15:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T03:15:10,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/b372e3f3947f425d9baa04f781f9ed64, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T03:15:10,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 42b167b16e2c28db0ea066b1d94c2c1f in 1355ms, sequenceid=60, compaction requested=true 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:10,316 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:10,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:10,316 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:10,318 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:10,318 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:10,318 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/B is initiating minor compaction (all files) 2024-11-17T03:15:10,318 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/A is initiating minor compaction (all files) 2024-11-17T03:15:10,318 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/B in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:10,318 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/A in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:10,318 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=90.7 K 2024-11-17T03:15:10,318 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d3e11553da0a40d0b1799e8cfdf06016, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/7883c07793054b08a9b76d99a35b223f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b5832965f8ac48bc967935fdaea1c7d3] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=35.2 K 2024-11-17T03:15:10,318 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:10,319 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc] 2024-11-17T03:15:10,320 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e11553da0a40d0b1799e8cfdf06016, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731813306041 2024-11-17T03:15:10,320 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc5c58991d4243f9942584d0e3c6efa1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731813306041 2024-11-17T03:15:10,321 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7883c07793054b08a9b76d99a35b223f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731813306126 2024-11-17T03:15:10,321 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 456ad50e120640df96a9bc1ab8de01c0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731813306126 2024-11-17T03:15:10,321 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b5832965f8ac48bc967935fdaea1c7d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731813308314 2024-11-17T03:15:10,322 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf874877c394cabab2edd2a0e91c1dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731813308314 2024-11-17T03:15:10,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T03:15:10,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:10,355 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:15:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:10,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:10,370 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:10,370 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:10,371 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/df48b2f468394be8b209970547935aba is 50, key is test_row_0/B:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:10,380 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173ce630063a5247efab1371afb93cbbc8_42b167b16e2c28db0ea066b1d94c2c1f store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:10,391 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173ce630063a5247efab1371afb93cbbc8_42b167b16e2c28db0ea066b1d94c2c1f, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:10,391 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173ce630063a5247efab1371afb93cbbc8_42b167b16e2c28db0ea066b1d94c2c1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111766e638e9c0bf40c6aee2d93bc74dfe08_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813308979/Put/seqid=0 2024-11-17T03:15:10,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741963_1139 (size=12104) 2024-11-17T03:15:10,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741965_1141 (size=12154) 2024-11-17T03:15:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741964_1140 (size=4469) 2024-11-17T03:15:10,442 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#A#compaction#121 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:10,442 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/21681a6a29b442a2b5e50f1fe4fc96f7 is 175, key is test_row_0/A:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:10,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741966_1142 (size=31058) 2024-11-17T03:15:10,475 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/21681a6a29b442a2b5e50f1fe4fc96f7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/21681a6a29b442a2b5e50f1fe4fc96f7 2024-11-17T03:15:10,484 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/A of 42b167b16e2c28db0ea066b1d94c2c1f into 21681a6a29b442a2b5e50f1fe4fc96f7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:10,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:10,484 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/A, priority=13, startTime=1731813310316; duration=0sec 2024-11-17T03:15:10,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:10,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:A 2024-11-17T03:15:10,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:10,485 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:10,485 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/C is initiating minor compaction (all files) 2024-11-17T03:15:10,485 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/C in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:10,486 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b1b3542c43d40d1a38df159fe44706a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/6cd5ca95c285465fac6f2404213ed810, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/b372e3f3947f425d9baa04f781f9ed64] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=35.2 K 2024-11-17T03:15:10,487 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b1b3542c43d40d1a38df159fe44706a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731813306041 2024-11-17T03:15:10,488 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd5ca95c285465fac6f2404213ed810, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731813306126 2024-11-17T03:15:10,488 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b372e3f3947f425d9baa04f781f9ed64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731813308314 2024-11-17T03:15:10,499 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#C#compaction#123 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:10,521 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/96033aa6ca0a47d88eaf57fed03bc3ea is 50, key is test_row_0/C:col10/1731813308322/Put/seqid=0 2024-11-17T03:15:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T03:15:10,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741967_1143 (size=12104) 2024-11-17T03:15:10,816 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/df48b2f468394be8b209970547935aba as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/df48b2f468394be8b209970547935aba 2024-11-17T03:15:10,822 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/B of 42b167b16e2c28db0ea066b1d94c2c1f into df48b2f468394be8b209970547935aba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:10,822 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:10,822 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/B, priority=13, startTime=1731813310316; duration=0sec 2024-11-17T03:15:10,822 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:10,822 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:B 2024-11-17T03:15:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T03:15:10,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:10,840 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111766e638e9c0bf40c6aee2d93bc74dfe08_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111766e638e9c0bf40c6aee2d93bc74dfe08_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/155d7dae42f9469e83181ee94e4335f1, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/155d7dae42f9469e83181ee94e4335f1 is 175, key is test_row_0/A:col10/1731813308979/Put/seqid=0 2024-11-17T03:15:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741968_1144 (size=30955) 2024-11-17T03:15:10,938 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/96033aa6ca0a47d88eaf57fed03bc3ea as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/96033aa6ca0a47d88eaf57fed03bc3ea 2024-11-17T03:15:10,950 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/C of 42b167b16e2c28db0ea066b1d94c2c1f into 96033aa6ca0a47d88eaf57fed03bc3ea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:10,950 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:10,950 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/C, priority=13, startTime=1731813310316; duration=0sec 2024-11-17T03:15:10,950 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:10,950 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:C 2024-11-17T03:15:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:11,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813371157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,163 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in 42b167b16e2c28db0ea066b1d94c2c1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:11,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813371159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813371159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813371161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813371163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813371267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813371267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813371267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,275 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/155d7dae42f9469e83181ee94e4335f1 2024-11-17T03:15:11,279 INFO [master/695c90f4bb18:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T03:15:11,280 INFO [master/695c90f4bb18:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T03:15:11,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813371277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/598f64e0af5f4a24a4f9c31e762ef236 is 50, key is test_row_0/B:col10/1731813308979/Put/seqid=0 2024-11-17T03:15:11,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741969_1145 (size=12001) 2024-11-17T03:15:11,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T03:15:11,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813371477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813371477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813371477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813371487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,697 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/598f64e0af5f4a24a4f9c31e762ef236 2024-11-17T03:15:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/19f638379a5148d5bbc12a27fe3c8089 is 50, key is test_row_0/C:col10/1731813308979/Put/seqid=0 2024-11-17T03:15:11,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741970_1146 (size=12001) 2024-11-17T03:15:11,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813371786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813371787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813371787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813371797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,115 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/19f638379a5148d5bbc12a27fe3c8089 2024-11-17T03:15:12,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/155d7dae42f9469e83181ee94e4335f1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1 2024-11-17T03:15:12,133 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1, entries=150, sequenceid=78, filesize=30.2 K 2024-11-17T03:15:12,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/598f64e0af5f4a24a4f9c31e762ef236 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/598f64e0af5f4a24a4f9c31e762ef236 2024-11-17T03:15:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,145 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/598f64e0af5f4a24a4f9c31e762ef236, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T03:15:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/19f638379a5148d5bbc12a27fe3c8089 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/19f638379a5148d5bbc12a27fe3c8089 2024-11-17T03:15:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,153 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/19f638379a5148d5bbc12a27fe3c8089, entries=150, sequenceid=78, filesize=11.7 K 2024-11-17T03:15:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,155 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 42b167b16e2c28db0ea066b1d94c2c1f in 1799ms, sequenceid=78, compaction requested=false 2024-11-17T03:15:12,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:12,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T03:15:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T03:15:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T03:15:12,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9550 sec 2024-11-17T03:15:12,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9620 sec 2024-11-17T03:15:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:15:12,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:12,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:12,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c1865df5e09243e88820eca68f41471d_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741971_1147 (size=12154) 2024-11-17T03:15:12,342 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T03:15:12,347 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:12,348 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:12,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c1865df5e09243e88820eca68f41471d_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c1865df5e09243e88820eca68f41471d_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T03:15:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813372343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/7978a1024528411c8e7c927723194c67, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:12,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/7978a1024528411c8e7c927723194c67 is 175, key is test_row_0/A:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:12,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,353 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:12,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:12,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813372346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813372346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813372347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741972_1148 (size=30951) 2024-11-17T03:15:12,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/7978a1024528411c8e7c927723194c67 2024-11-17T03:15:12,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/aa2a399ea4ec497e899cdd4ff2b0daf8 is 50, key is test_row_0/B:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:12,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741973_1149 (size=9657) 2024-11-17T03:15:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813372457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813372466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813372466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813372466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T03:15:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T03:15:12,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813372667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813372677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813372677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813372677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/aa2a399ea4ec497e899cdd4ff2b0daf8 2024-11-17T03:15:12,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/cdaad50ba210426090c8d9d1f0b2ffc1 is 50, key is test_row_0/C:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:12,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741974_1150 (size=9657) 2024-11-17T03:15:12,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T03:15:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T03:15:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:12,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813372977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813372986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813372987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813372987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T03:15:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:13,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813373187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,189 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in 42b167b16e2c28db0ea066b1d94c2c1f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/cdaad50ba210426090c8d9d1f0b2ffc1 2024-11-17T03:15:13,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/7978a1024528411c8e7c927723194c67 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67 2024-11-17T03:15:13,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67, entries=150, sequenceid=96, filesize=30.2 K 2024-11-17T03:15:13,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/aa2a399ea4ec497e899cdd4ff2b0daf8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/aa2a399ea4ec497e899cdd4ff2b0daf8 2024-11-17T03:15:13,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/aa2a399ea4ec497e899cdd4ff2b0daf8, entries=100, sequenceid=96, filesize=9.4 K 2024-11-17T03:15:13,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/cdaad50ba210426090c8d9d1f0b2ffc1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cdaad50ba210426090c8d9d1f0b2ffc1 2024-11-17T03:15:13,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cdaad50ba210426090c8d9d1f0b2ffc1, entries=100, sequenceid=96, filesize=9.4 K 2024-11-17T03:15:13,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 42b167b16e2c28db0ea066b1d94c2c1f in 922ms, sequenceid=96, compaction requested=true 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:13,227 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:13,227 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:13,228 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:13,228 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:13,228 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/A is initiating minor compaction (all files) 2024-11-17T03:15:13,228 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/B is initiating minor compaction (all files) 2024-11-17T03:15:13,229 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/B in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,229 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/A in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,229 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/21681a6a29b442a2b5e50f1fe4fc96f7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=90.8 K 2024-11-17T03:15:13,229 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/df48b2f468394be8b209970547935aba, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/598f64e0af5f4a24a4f9c31e762ef236, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/aa2a399ea4ec497e899cdd4ff2b0daf8] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=33.0 K 2024-11-17T03:15:13,229 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,229 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/21681a6a29b442a2b5e50f1fe4fc96f7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67] 2024-11-17T03:15:13,229 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting df48b2f468394be8b209970547935aba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731813308314 2024-11-17T03:15:13,229 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21681a6a29b442a2b5e50f1fe4fc96f7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731813308314 2024-11-17T03:15:13,230 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 598f64e0af5f4a24a4f9c31e762ef236, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731813308963 2024-11-17T03:15:13,230 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155d7dae42f9469e83181ee94e4335f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731813308963 2024-11-17T03:15:13,230 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2a399ea4ec497e899cdd4ff2b0daf8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813311161 2024-11-17T03:15:13,230 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7978a1024528411c8e7c927723194c67, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813311158 2024-11-17T03:15:13,235 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:13,238 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:13,238 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a36a5307c5b246c9bab93fb895aac3d5_42b167b16e2c28db0ea066b1d94c2c1f store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:13,238 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/9fc3131a641d4a8eb9279471d4a14d2a is 50, key is test_row_0/B:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:13,240 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a36a5307c5b246c9bab93fb895aac3d5_42b167b16e2c28db0ea066b1d94c2c1f, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:13,240 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a36a5307c5b246c9bab93fb895aac3d5_42b167b16e2c28db0ea066b1d94c2c1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:13,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741976_1152 (size=4469) 2024-11-17T03:15:13,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741975_1151 (size=12207) 2024-11-17T03:15:13,248 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#A#compaction#129 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:13,249 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/6127742709fd4d5c8caa50db46e2d3e3 is 175, key is test_row_0/A:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:13,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741977_1153 (size=31268) 2024-11-17T03:15:13,260 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/6127742709fd4d5c8caa50db46e2d3e3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/6127742709fd4d5c8caa50db46e2d3e3 2024-11-17T03:15:13,265 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/A of 42b167b16e2c28db0ea066b1d94c2c1f into 6127742709fd4d5c8caa50db46e2d3e3(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:13,265 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:13,265 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/A, priority=13, startTime=1731813313227; duration=0sec 2024-11-17T03:15:13,265 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:13,266 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:A 2024-11-17T03:15:13,266 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:13,267 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:13,267 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/C is initiating minor compaction (all files) 2024-11-17T03:15:13,267 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/C in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,267 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/96033aa6ca0a47d88eaf57fed03bc3ea, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/19f638379a5148d5bbc12a27fe3c8089, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cdaad50ba210426090c8d9d1f0b2ffc1] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=33.0 K 2024-11-17T03:15:13,267 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96033aa6ca0a47d88eaf57fed03bc3ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731813308314 2024-11-17T03:15:13,269 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f638379a5148d5bbc12a27fe3c8089, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731813308963 2024-11-17T03:15:13,270 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdaad50ba210426090c8d9d1f0b2ffc1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813311161 2024-11-17T03:15:13,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T03:15:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:13,271 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:15:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:13,281 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:13,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fc5ab00a0ab04959b2c393f0bded3fb2_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813312341/Put/seqid=0 2024-11-17T03:15:13,281 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/5edcdad76d1740ea96525907cdb64d81 is 50, key is test_row_0/C:col10/1731813311161/Put/seqid=0 2024-11-17T03:15:13,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741978_1154 (size=12207) 2024-11-17T03:15:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741979_1155 (size=12154) 2024-11-17T03:15:13,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:13,296 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fc5ab00a0ab04959b2c393f0bded3fb2_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fc5ab00a0ab04959b2c393f0bded3fb2_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:13,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/0257dda9b0e74f9a87f51361bd000b0e, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:13,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/0257dda9b0e74f9a87f51361bd000b0e is 175, key is test_row_0/A:col10/1731813312341/Put/seqid=0 2024-11-17T03:15:13,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741980_1156 (size=30955) 2024-11-17T03:15:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:13,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813373507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813373508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813373509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813373510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813373617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813373617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813373617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813373627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,652 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/9fc3131a641d4a8eb9279471d4a14d2a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/9fc3131a641d4a8eb9279471d4a14d2a 2024-11-17T03:15:13,657 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/B of 42b167b16e2c28db0ea066b1d94c2c1f into 9fc3131a641d4a8eb9279471d4a14d2a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:13,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:13,657 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/B, priority=13, startTime=1731813313227; duration=0sec 2024-11-17T03:15:13,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:13,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:B 2024-11-17T03:15:13,692 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/5edcdad76d1740ea96525907cdb64d81 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/5edcdad76d1740ea96525907cdb64d81 2024-11-17T03:15:13,697 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/C of 42b167b16e2c28db0ea066b1d94c2c1f into 5edcdad76d1740ea96525907cdb64d81(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:13,697 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:13,697 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/C, priority=13, startTime=1731813313227; duration=0sec 2024-11-17T03:15:13,697 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:13,697 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:C 2024-11-17T03:15:13,702 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/0257dda9b0e74f9a87f51361bd000b0e 2024-11-17T03:15:13,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/de8a53a643ac45dca6f16ba02d129a8f is 50, key is test_row_0/B:col10/1731813312341/Put/seqid=0 2024-11-17T03:15:13,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741981_1157 (size=12001) 2024-11-17T03:15:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813373827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813373827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813373827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813373837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,117 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/de8a53a643ac45dca6f16ba02d129a8f 2024-11-17T03:15:14,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/cafcb883a4fc4623bb128223d51ae7e4 is 50, key is test_row_0/C:col10/1731813312341/Put/seqid=0 2024-11-17T03:15:14,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741982_1158 (size=12001) 2024-11-17T03:15:14,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813374137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813374138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813374138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813374147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:14,536 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/cafcb883a4fc4623bb128223d51ae7e4 2024-11-17T03:15:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/0257dda9b0e74f9a87f51361bd000b0e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e 2024-11-17T03:15:14,544 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e, entries=150, sequenceid=114, filesize=30.2 K 2024-11-17T03:15:14,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/de8a53a643ac45dca6f16ba02d129a8f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/de8a53a643ac45dca6f16ba02d129a8f 2024-11-17T03:15:14,549 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/de8a53a643ac45dca6f16ba02d129a8f, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T03:15:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/cafcb883a4fc4623bb128223d51ae7e4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cafcb883a4fc4623bb128223d51ae7e4 2024-11-17T03:15:14,554 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cafcb883a4fc4623bb128223d51ae7e4, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T03:15:14,555 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 42b167b16e2c28db0ea066b1d94c2c1f in 1284ms, sequenceid=114, compaction requested=false 2024-11-17T03:15:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:14,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T03:15:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T03:15:14,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T03:15:14,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2030 sec 2024-11-17T03:15:14,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2100 sec 2024-11-17T03:15:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:14,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T03:15:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:14,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:14,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111733e6271054594a4f836ee714f68651ae_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:14,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741983_1159 (size=14644) 2024-11-17T03:15:14,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:14,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111733e6271054594a4f836ee714f68651ae_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111733e6271054594a4f836ee714f68651ae_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:14,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813374664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813374665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/8e23716d3d8344c99de53e6e3868b8eb, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:14,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/8e23716d3d8344c99de53e6e3868b8eb is 175, key is test_row_0/A:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813374667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813374668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741984_1160 (size=39599) 2024-11-17T03:15:14,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813374987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813374987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813374987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813374987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/8e23716d3d8344c99de53e6e3868b8eb 2024-11-17T03:15:15,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/d2333150c5cb4e01a88ec7d8c9a0580f is 50, key is test_row_0/B:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:15,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741985_1161 (size=12051) 2024-11-17T03:15:15,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813375297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813375297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813375297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813375297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/d2333150c5cb4e01a88ec7d8c9a0580f 2024-11-17T03:15:15,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/930e21d74ad34bb9a130f63c9bdc540c is 50, key is test_row_0/C:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741986_1162 (size=12051) 2024-11-17T03:15:15,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T03:15:15,663 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T03:15:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813375807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813375807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813375807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813375807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:15,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/930e21d74ad34bb9a130f63c9bdc540c 2024-11-17T03:15:15,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/8e23716d3d8344c99de53e6e3868b8eb as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb 2024-11-17T03:15:15,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb, entries=200, sequenceid=132, filesize=38.7 K 2024-11-17T03:15:15,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/d2333150c5cb4e01a88ec7d8c9a0580f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d2333150c5cb4e01a88ec7d8c9a0580f 2024-11-17T03:15:15,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d2333150c5cb4e01a88ec7d8c9a0580f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T03:15:15,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/930e21d74ad34bb9a130f63c9bdc540c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/930e21d74ad34bb9a130f63c9bdc540c 2024-11-17T03:15:15,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/930e21d74ad34bb9a130f63c9bdc540c, entries=150, sequenceid=132, filesize=11.8 K 2024-11-17T03:15:15,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 42b167b16e2c28db0ea066b1d94c2c1f in 1275ms, sequenceid=132, compaction requested=true 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:15,924 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:15,924 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:15,925 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:15,925 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101822 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:15,925 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/B is initiating minor compaction (all files) 2024-11-17T03:15:15,925 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/A is initiating minor compaction (all files) 2024-11-17T03:15:15,925 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/B in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:15,925 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/A in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:15,925 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/6127742709fd4d5c8caa50db46e2d3e3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=99.4 K 2024-11-17T03:15:15,925 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/9fc3131a641d4a8eb9279471d4a14d2a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/de8a53a643ac45dca6f16ba02d129a8f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d2333150c5cb4e01a88ec7d8c9a0580f] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=35.4 K 2024-11-17T03:15:15,925 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:15,925 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/6127742709fd4d5c8caa50db46e2d3e3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb] 2024-11-17T03:15:15,926 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc3131a641d4a8eb9279471d4a14d2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813308963 2024-11-17T03:15:15,926 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6127742709fd4d5c8caa50db46e2d3e3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813308963 2024-11-17T03:15:15,926 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0257dda9b0e74f9a87f51361bd000b0e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731813312341 2024-11-17T03:15:15,926 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting de8a53a643ac45dca6f16ba02d129a8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731813312341 2024-11-17T03:15:15,926 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e23716d3d8344c99de53e6e3868b8eb, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731813313501 2024-11-17T03:15:15,926 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting d2333150c5cb4e01a88ec7d8c9a0580f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731813313501 2024-11-17T03:15:15,933 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:15,935 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:15,935 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/c6c8ca6c9422475cb3cecf5c44c8f314 is 50, key is test_row_0/B:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:15,939 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111773115900f2db47cc931e566f7d51f974_42b167b16e2c28db0ea066b1d94c2c1f store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:15,940 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111773115900f2db47cc931e566f7d51f974_42b167b16e2c28db0ea066b1d94c2c1f, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:15,941 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111773115900f2db47cc931e566f7d51f974_42b167b16e2c28db0ea066b1d94c2c1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:15,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741987_1163 (size=12359) 2024-11-17T03:15:15,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741988_1164 (size=4469) 2024-11-17T03:15:16,357 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/c6c8ca6c9422475cb3cecf5c44c8f314 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/c6c8ca6c9422475cb3cecf5c44c8f314 2024-11-17T03:15:16,362 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/B of 42b167b16e2c28db0ea066b1d94c2c1f into c6c8ca6c9422475cb3cecf5c44c8f314(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:16,362 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:16,362 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/B, priority=13, startTime=1731813315924; duration=0sec 2024-11-17T03:15:16,362 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:16,362 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:B 2024-11-17T03:15:16,363 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:16,364 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:16,364 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/C is initiating minor compaction (all files) 2024-11-17T03:15:16,364 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/C in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:16,364 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/5edcdad76d1740ea96525907cdb64d81, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cafcb883a4fc4623bb128223d51ae7e4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/930e21d74ad34bb9a130f63c9bdc540c] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=35.4 K 2024-11-17T03:15:16,365 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edcdad76d1740ea96525907cdb64d81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813308963 2024-11-17T03:15:16,366 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting cafcb883a4fc4623bb128223d51ae7e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731813312341 2024-11-17T03:15:16,366 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#A#compaction#138 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:16,367 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/fe023372c5924d198576711a5a04c0fd is 175, key is test_row_0/A:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:16,368 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 930e21d74ad34bb9a130f63c9bdc540c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731813313501 2024-11-17T03:15:16,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741989_1165 (size=31313) 2024-11-17T03:15:16,375 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:16,376 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/2ad6beac8f9741d28382fc0d2e2c9111 is 50, key is test_row_0/C:col10/1731813314647/Put/seqid=0 2024-11-17T03:15:16,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741990_1166 (size=12359) 2024-11-17T03:15:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T03:15:16,507 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:16,507 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T03:15:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T03:15:16,510 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:16,511 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:16,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T03:15:16,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T03:15:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:16,664 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T03:15:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:16,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f7f521d6db946819a01f2491b19abf8_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813314665/Put/seqid=0 2024-11-17T03:15:16,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741991_1167 (size=12304) 2024-11-17T03:15:16,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:16,684 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f7f521d6db946819a01f2491b19abf8_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f7f521d6db946819a01f2491b19abf8_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:16,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/4cce27b08f804bb6a4995f4a78db1e56, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:16,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/4cce27b08f804bb6a4995f4a78db1e56 is 175, key is test_row_0/A:col10/1731813314665/Put/seqid=0 2024-11-17T03:15:16,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741992_1168 (size=31105) 2024-11-17T03:15:16,777 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/fe023372c5924d198576711a5a04c0fd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fe023372c5924d198576711a5a04c0fd 2024-11-17T03:15:16,782 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/A of 42b167b16e2c28db0ea066b1d94c2c1f into fe023372c5924d198576711a5a04c0fd(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:16,783 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:16,783 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/A, priority=13, startTime=1731813315924; duration=0sec 2024-11-17T03:15:16,783 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:16,783 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:A 2024-11-17T03:15:16,788 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/2ad6beac8f9741d28382fc0d2e2c9111 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/2ad6beac8f9741d28382fc0d2e2c9111 2024-11-17T03:15:16,793 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/C of 42b167b16e2c28db0ea066b1d94c2c1f into 2ad6beac8f9741d28382fc0d2e2c9111(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:16,793 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:16,793 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/C, priority=13, startTime=1731813315924; duration=0sec 2024-11-17T03:15:16,793 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:16,793 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:C 2024-11-17T03:15:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:16,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T03:15:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813376834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813376834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813376836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813376836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813376947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813376947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813376947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813376947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:16,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,090 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/4cce27b08f804bb6a4995f4a78db1e56 2024-11-17T03:15:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T03:15:17,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/20a8b254e6de4bcdab6436621d2e7fa2 is 50, key is test_row_0/B:col10/1731813314665/Put/seqid=0 2024-11-17T03:15:17,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741993_1169 (size=12151) 2024-11-17T03:15:17,144 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/20a8b254e6de4bcdab6436621d2e7fa2 2024-11-17T03:15:17,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/e528b81c8c244b54b2797770a62d2deb is 50, key is test_row_0/C:col10/1731813314665/Put/seqid=0 2024-11-17T03:15:17,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741994_1170 (size=12151) 2024-11-17T03:15:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813377157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813377157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813377158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813377158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41498 deadline: 1731813377217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,218 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in 42b167b16e2c28db0ea066b1d94c2c1f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813377467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813377467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813377467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813377467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,556 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/e528b81c8c244b54b2797770a62d2deb 2024-11-17T03:15:17,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/4cce27b08f804bb6a4995f4a78db1e56 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56 2024-11-17T03:15:17,565 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56, entries=150, sequenceid=149, filesize=30.4 K 2024-11-17T03:15:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/20a8b254e6de4bcdab6436621d2e7fa2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/20a8b254e6de4bcdab6436621d2e7fa2 2024-11-17T03:15:17,574 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/20a8b254e6de4bcdab6436621d2e7fa2, entries=150, sequenceid=149, filesize=11.9 K 2024-11-17T03:15:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/e528b81c8c244b54b2797770a62d2deb as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/e528b81c8c244b54b2797770a62d2deb 2024-11-17T03:15:17,579 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/e528b81c8c244b54b2797770a62d2deb, entries=150, sequenceid=149, filesize=11.9 K 2024-11-17T03:15:17,580 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 42b167b16e2c28db0ea066b1d94c2c1f in 916ms, sequenceid=149, compaction requested=false 2024-11-17T03:15:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T03:15:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T03:15:17,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T03:15:17,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-17T03:15:17,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0770 sec 2024-11-17T03:15:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T03:15:17,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:17,647 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:17,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T03:15:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T03:15:17,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:17,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:17,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T03:15:17,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T03:15:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:17,802 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T03:15:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:17,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:17,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:17,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d2cffe4823c2417b8ef25eba7b4061ce_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813316826/Put/seqid=0 2024-11-17T03:15:17,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741995_1171 (size=12304) 2024-11-17T03:15:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T03:15:17,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:17,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813377994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:17,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813377996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813377996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813377998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813378107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813378107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813378107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813378107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:18,224 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d2cffe4823c2417b8ef25eba7b4061ce_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d2cffe4823c2417b8ef25eba7b4061ce_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/00013ed60a3044b58e59d4090172a977, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:18,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/00013ed60a3044b58e59d4090172a977 is 175, key is test_row_0/A:col10/1731813316826/Put/seqid=0 2024-11-17T03:15:18,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741996_1172 (size=31105) 2024-11-17T03:15:18,230 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/00013ed60a3044b58e59d4090172a977 2024-11-17T03:15:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/e89187d02cb94a99a3d0fae3b235c57f is 50, key is test_row_0/B:col10/1731813316826/Put/seqid=0 2024-11-17T03:15:18,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741997_1173 (size=12151) 2024-11-17T03:15:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T03:15:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813378317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813378627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813378627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813378627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813378627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:18,642 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/e89187d02cb94a99a3d0fae3b235c57f 2024-11-17T03:15:18,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/259ac477a5344e91bd93a67929ced4fa is 50, key is test_row_0/C:col10/1731813316826/Put/seqid=0 2024-11-17T03:15:18,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741998_1174 (size=12151) 2024-11-17T03:15:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T03:15:19,058 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/259ac477a5344e91bd93a67929ced4fa 2024-11-17T03:15:19,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/00013ed60a3044b58e59d4090172a977 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977 2024-11-17T03:15:19,067 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977, entries=150, sequenceid=167, filesize=30.4 K 2024-11-17T03:15:19,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/e89187d02cb94a99a3d0fae3b235c57f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e89187d02cb94a99a3d0fae3b235c57f 2024-11-17T03:15:19,073 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e89187d02cb94a99a3d0fae3b235c57f, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T03:15:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/259ac477a5344e91bd93a67929ced4fa as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/259ac477a5344e91bd93a67929ced4fa 2024-11-17T03:15:19,078 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/259ac477a5344e91bd93a67929ced4fa, entries=150, sequenceid=167, filesize=11.9 K 2024-11-17T03:15:19,079 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 42b167b16e2c28db0ea066b1d94c2c1f in 1277ms, sequenceid=167, compaction requested=true 2024-11-17T03:15:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T03:15:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T03:15:19,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T03:15:19,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-17T03:15:19,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4350 sec 2024-11-17T03:15:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:19,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:15:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:19,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:19,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c32d256a13ae450e8e42e8c5ac5cf7bf_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:19,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741999_1175 (size=17284) 2024-11-17T03:15:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813379158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813379154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813379159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813379160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:19,561 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c32d256a13ae450e8e42e8c5ac5cf7bf_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c32d256a13ae450e8e42e8c5ac5cf7bf_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:19,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/e3f2c15ffaad466c82d9e9fede8f9f29, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:19,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/e3f2c15ffaad466c82d9e9fede8f9f29 is 175, key is test_row_0/A:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:19,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742000_1176 (size=48389) 2024-11-17T03:15:19,565 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/e3f2c15ffaad466c82d9e9fede8f9f29 2024-11-17T03:15:19,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/e32aad34a3634c559da631036c4dec71 is 50, key is test_row_0/B:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:19,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742001_1177 (size=12151) 2024-11-17T03:15:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813379786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813379786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813379786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813379787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T03:15:19,797 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:19,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T03:15:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T03:15:19,799 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:19,800 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:19,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T03:15:19,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T03:15:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:19,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:19,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/e32aad34a3634c559da631036c4dec71 2024-11-17T03:15:19,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/4c3880ce2ee844b389564a742948283a is 50, key is test_row_0/C:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742002_1178 (size=12151) 2024-11-17T03:15:20,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T03:15:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T03:15:20,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T03:15:20,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:20,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813380297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813380297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813380297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:20,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813380297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:20,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/4c3880ce2ee844b389564a742948283a 2024-11-17T03:15:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/e3f2c15ffaad466c82d9e9fede8f9f29 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29 2024-11-17T03:15:20,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29, entries=250, sequenceid=187, filesize=47.3 K 2024-11-17T03:15:20,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/e32aad34a3634c559da631036c4dec71 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e32aad34a3634c559da631036c4dec71 2024-11-17T03:15:20,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e32aad34a3634c559da631036c4dec71, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T03:15:20,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/4c3880ce2ee844b389564a742948283a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4c3880ce2ee844b389564a742948283a 2024-11-17T03:15:20,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T03:15:20,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:20,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4c3880ce2ee844b389564a742948283a, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T03:15:20,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 42b167b16e2c28db0ea066b1d94c2c1f in 1271ms, sequenceid=187, compaction requested=true 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:20,413 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:20,413 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:20,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:20,415 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:20,415 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/A is initiating minor compaction (all files) 2024-11-17T03:15:20,415 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/A in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,415 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:20,415 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/B is initiating minor compaction (all files) 2024-11-17T03:15:20,415 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fe023372c5924d198576711a5a04c0fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=138.6 K 2024-11-17T03:15:20,415 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/B in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,415 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,415 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/c6c8ca6c9422475cb3cecf5c44c8f314, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/20a8b254e6de4bcdab6436621d2e7fa2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e89187d02cb94a99a3d0fae3b235c57f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e32aad34a3634c559da631036c4dec71] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=47.7 K 2024-11-17T03:15:20,415 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fe023372c5924d198576711a5a04c0fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29] 2024-11-17T03:15:20,416 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe023372c5924d198576711a5a04c0fd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731813313501 2024-11-17T03:15:20,416 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c8ca6c9422475cb3cecf5c44c8f314, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731813313501 2024-11-17T03:15:20,416 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a8b254e6de4bcdab6436621d2e7fa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731813314664 2024-11-17T03:15:20,416 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cce27b08f804bb6a4995f4a78db1e56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731813314664 2024-11-17T03:15:20,416 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e89187d02cb94a99a3d0fae3b235c57f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731813316826 2024-11-17T03:15:20,416 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00013ed60a3044b58e59d4090172a977, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731813316826 2024-11-17T03:15:20,417 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e32aad34a3634c559da631036c4dec71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731813317994 2024-11-17T03:15:20,417 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3f2c15ffaad466c82d9e9fede8f9f29, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731813317994 2024-11-17T03:15:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T03:15:20,431 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:20,433 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e7be30f0a1af4a5b9b44675c3e650af2_42b167b16e2c28db0ea066b1d94c2c1f store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:20,433 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:20,434 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/b97ac7bb9cb74fde87ba1fd1c359075e is 50, key is test_row_0/B:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:20,436 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e7be30f0a1af4a5b9b44675c3e650af2_42b167b16e2c28db0ea066b1d94c2c1f, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:20,436 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e7be30f0a1af4a5b9b44675c3e650af2_42b167b16e2c28db0ea066b1d94c2c1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:20,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742003_1179 (size=12595) 2024-11-17T03:15:20,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742004_1180 (size=4469) 2024-11-17T03:15:20,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T03:15:20,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,566 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:15:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:20,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:20,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174ece741bf4a845c08c5c67cd20db3fda_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813319157/Put/seqid=0 2024-11-17T03:15:20,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742005_1181 (size=12304) 2024-11-17T03:15:20,843 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/b97ac7bb9cb74fde87ba1fd1c359075e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b97ac7bb9cb74fde87ba1fd1c359075e 2024-11-17T03:15:20,847 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:20,848 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/3d9c50cd70f74cd39ecbd684cbb1de1f is 175, key is test_row_0/A:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:20,848 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/B of 42b167b16e2c28db0ea066b1d94c2c1f into b97ac7bb9cb74fde87ba1fd1c359075e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:20,848 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:20,848 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/B, priority=12, startTime=1731813320413; duration=0sec 2024-11-17T03:15:20,849 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:20,849 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:B 2024-11-17T03:15:20,849 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:20,850 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:20,850 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/C is initiating minor compaction (all files) 2024-11-17T03:15:20,850 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/C in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:20,851 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/2ad6beac8f9741d28382fc0d2e2c9111, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/e528b81c8c244b54b2797770a62d2deb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/259ac477a5344e91bd93a67929ced4fa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4c3880ce2ee844b389564a742948283a] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=47.7 K 2024-11-17T03:15:20,851 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad6beac8f9741d28382fc0d2e2c9111, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731813313501 2024-11-17T03:15:20,851 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e528b81c8c244b54b2797770a62d2deb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731813314664 2024-11-17T03:15:20,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 259ac477a5344e91bd93a67929ced4fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731813316826 2024-11-17T03:15:20,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c3880ce2ee844b389564a742948283a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731813317994 2024-11-17T03:15:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742006_1182 (size=31549) 2024-11-17T03:15:20,861 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:20,861 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/1fd465c77cf74058a4adbd7353b92e47 is 50, key is test_row_0/C:col10/1731813319142/Put/seqid=0 2024-11-17T03:15:20,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742007_1183 (size=12595) 2024-11-17T03:15:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T03:15:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:20,983 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174ece741bf4a845c08c5c67cd20db3fda_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174ece741bf4a845c08c5c67cd20db3fda_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:20,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/2529d2ff37254a2a87f8d5935a56225a, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:20,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/2529d2ff37254a2a87f8d5935a56225a is 175, key is test_row_0/A:col10/1731813319157/Put/seqid=0 2024-11-17T03:15:20,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742008_1184 (size=31105) 2024-11-17T03:15:20,994 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/2529d2ff37254a2a87f8d5935a56225a 2024-11-17T03:15:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/8fd9ccc01cd1433cb3572da56f7683d9 is 50, key is test_row_0/B:col10/1731813319157/Put/seqid=0 2024-11-17T03:15:21,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742009_1185 (size=12151) 2024-11-17T03:15:21,258 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/3d9c50cd70f74cd39ecbd684cbb1de1f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/3d9c50cd70f74cd39ecbd684cbb1de1f 2024-11-17T03:15:21,263 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/A of 42b167b16e2c28db0ea066b1d94c2c1f into 3d9c50cd70f74cd39ecbd684cbb1de1f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:21,263 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:21,263 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/A, priority=12, startTime=1731813320413; duration=0sec 2024-11-17T03:15:21,263 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:21,263 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:A 2024-11-17T03:15:21,287 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/1fd465c77cf74058a4adbd7353b92e47 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/1fd465c77cf74058a4adbd7353b92e47 2024-11-17T03:15:21,292 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/C of 42b167b16e2c28db0ea066b1d94c2c1f into 1fd465c77cf74058a4adbd7353b92e47(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:21,292 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:21,292 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/C, priority=12, startTime=1731813320413; duration=0sec 2024-11-17T03:15:21,292 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:21,292 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:C 2024-11-17T03:15:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:21,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813381326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813381326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813381327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813381329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,418 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/8fd9ccc01cd1433cb3572da56f7683d9 2024-11-17T03:15:21,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/8ec8483e866b4da08594992623c394cd is 50, key is test_row_0/C:col10/1731813319157/Put/seqid=0 2024-11-17T03:15:21,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742010_1186 (size=12151) 2024-11-17T03:15:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813381437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813381437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813381437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813381437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813381647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813381647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813381647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813381647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,833 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/8ec8483e866b4da08594992623c394cd 2024-11-17T03:15:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/2529d2ff37254a2a87f8d5935a56225a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a 2024-11-17T03:15:21,842 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a, entries=150, sequenceid=199, filesize=30.4 K 2024-11-17T03:15:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/8fd9ccc01cd1433cb3572da56f7683d9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/8fd9ccc01cd1433cb3572da56f7683d9 2024-11-17T03:15:21,848 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/8fd9ccc01cd1433cb3572da56f7683d9, entries=150, sequenceid=199, filesize=11.9 K 2024-11-17T03:15:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/8ec8483e866b4da08594992623c394cd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/8ec8483e866b4da08594992623c394cd 2024-11-17T03:15:21,854 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/8ec8483e866b4da08594992623c394cd, entries=150, sequenceid=199, filesize=11.9 K 2024-11-17T03:15:21,855 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 42b167b16e2c28db0ea066b1d94c2c1f in 1290ms, sequenceid=199, compaction requested=false 2024-11-17T03:15:21,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:21,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:21,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T03:15:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T03:15:21,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T03:15:21,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-17T03:15:21,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0620 sec 2024-11-17T03:15:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T03:15:21,947 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:21,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T03:15:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:21,949 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:21,949 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:21,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:21,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-17T03:15:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:21,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f6e6a1a3315f4381a13ab91a45aed535_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:21,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742011_1187 (size=12304) 2024-11-17T03:15:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813381968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813381969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813381971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813381971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813382077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813382077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813382086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813382087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:22,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:22,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813382287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813382287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813382296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813382297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,372 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:22,375 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f6e6a1a3315f4381a13ab91a45aed535_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f6e6a1a3315f4381a13ab91a45aed535_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:22,376 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/1e9f2835f5d346be870f4db1770da86e, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:22,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/1e9f2835f5d346be870f4db1770da86e is 175, key is test_row_0/A:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742012_1188 (size=31105) 2024-11-17T03:15:22,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:22,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813382597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813382597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813382607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813382607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:22,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/1e9f2835f5d346be870f4db1770da86e 2024-11-17T03:15:22,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/402eae963f1f484d874976bb3c993ca9 is 50, key is test_row_0/B:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742013_1189 (size=12151) 2024-11-17T03:15:22,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:22,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:22,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:22,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813383107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813383107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:23,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813383117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813383127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:23,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/402eae963f1f484d874976bb3c993ca9 2024-11-17T03:15:23,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/0b12e973073446e0b4c91aed1ba65cd4 is 50, key is test_row_0/C:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742014_1190 (size=12151) 2024-11-17T03:15:23,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:23,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:23,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/0b12e973073446e0b4c91aed1ba65cd4 2024-11-17T03:15:23,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/1e9f2835f5d346be870f4db1770da86e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e 2024-11-17T03:15:23,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e, entries=150, sequenceid=224, filesize=30.4 K 2024-11-17T03:15:23,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/402eae963f1f484d874976bb3c993ca9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/402eae963f1f484d874976bb3c993ca9 2024-11-17T03:15:23,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/402eae963f1f484d874976bb3c993ca9, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T03:15:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/0b12e973073446e0b4c91aed1ba65cd4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b12e973073446e0b4c91aed1ba65cd4 2024-11-17T03:15:23,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b12e973073446e0b4c91aed1ba65cd4, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T03:15:23,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 42b167b16e2c28db0ea066b1d94c2c1f in 1666ms, sequenceid=224, compaction requested=true 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:23,624 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:23,624 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:23,625 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:23,625 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:23,625 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/B is initiating minor compaction (all files) 2024-11-17T03:15:23,625 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/A is initiating minor compaction (all files) 2024-11-17T03:15:23,625 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/B in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,625 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/A in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,625 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b97ac7bb9cb74fde87ba1fd1c359075e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/8fd9ccc01cd1433cb3572da56f7683d9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/402eae963f1f484d874976bb3c993ca9] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=36.0 K 2024-11-17T03:15:23,625 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/3d9c50cd70f74cd39ecbd684cbb1de1f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=91.6 K 2024-11-17T03:15:23,625 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,625 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/3d9c50cd70f74cd39ecbd684cbb1de1f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e] 2024-11-17T03:15:23,625 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b97ac7bb9cb74fde87ba1fd1c359075e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731813317994 2024-11-17T03:15:23,626 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9c50cd70f74cd39ecbd684cbb1de1f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731813317994 2024-11-17T03:15:23,626 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd9ccc01cd1433cb3572da56f7683d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731813319149 2024-11-17T03:15:23,626 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 402eae963f1f484d874976bb3c993ca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731813321326 2024-11-17T03:15:23,626 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2529d2ff37254a2a87f8d5935a56225a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731813319149 2024-11-17T03:15:23,626 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e9f2835f5d346be870f4db1770da86e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731813321326 2024-11-17T03:15:23,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T03:15:23,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,629 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:23,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:23,640 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:23,642 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:23,642 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/4c89da967f3f42bead9bc0e185a1e5c9 is 50, key is test_row_0/B:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:23,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a87104650f8746eda055f397ea7d4cd3_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813321969/Put/seqid=0 2024-11-17T03:15:23,645 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411177f10b35da9c94066ba1fceaca6fe4649_42b167b16e2c28db0ea066b1d94c2c1f store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:23,647 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411177f10b35da9c94066ba1fceaca6fe4649_42b167b16e2c28db0ea066b1d94c2c1f, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:23,647 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177f10b35da9c94066ba1fceaca6fe4649_42b167b16e2c28db0ea066b1d94c2c1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:23,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742016_1192 (size=12304) 2024-11-17T03:15:23,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742015_1191 (size=12697) 2024-11-17T03:15:23,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742017_1193 (size=4469) 2024-11-17T03:15:23,665 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/4c89da967f3f42bead9bc0e185a1e5c9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/4c89da967f3f42bead9bc0e185a1e5c9 2024-11-17T03:15:23,669 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/B of 42b167b16e2c28db0ea066b1d94c2c1f into 4c89da967f3f42bead9bc0e185a1e5c9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:23,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:23,670 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/B, priority=13, startTime=1731813323624; duration=0sec 2024-11-17T03:15:23,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:23,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:B 2024-11-17T03:15:23,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:23,671 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:23,671 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/C is initiating minor compaction (all files) 2024-11-17T03:15:23,671 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/C in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:23,671 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/1fd465c77cf74058a4adbd7353b92e47, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/8ec8483e866b4da08594992623c394cd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b12e973073446e0b4c91aed1ba65cd4] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=36.0 K 2024-11-17T03:15:23,671 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fd465c77cf74058a4adbd7353b92e47, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731813317994 2024-11-17T03:15:23,672 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec8483e866b4da08594992623c394cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731813319149 2024-11-17T03:15:23,672 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b12e973073446e0b4c91aed1ba65cd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731813321326 2024-11-17T03:15:23,680 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:23,680 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/67d54670e4ee4d19b2d40f166456b051 is 50, key is test_row_0/C:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:23,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742018_1194 (size=12697) 2024-11-17T03:15:24,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:24,060 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a87104650f8746eda055f397ea7d4cd3_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a87104650f8746eda055f397ea7d4cd3_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:24,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/46d6cc447cec4727b37412dcc14d0ce6, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:24,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/46d6cc447cec4727b37412dcc14d0ce6 is 175, key is test_row_0/A:col10/1731813321969/Put/seqid=0 2024-11-17T03:15:24,062 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:24,063 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/9b08f3ee569747e4a6179d89d0711f4a is 175, key is test_row_0/A:col10/1731813321957/Put/seqid=0 2024-11-17T03:15:24,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742019_1195 (size=31105) 2024-11-17T03:15:24,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742020_1196 (size=31651) 2024-11-17T03:15:24,090 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/67d54670e4ee4d19b2d40f166456b051 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/67d54670e4ee4d19b2d40f166456b051 2024-11-17T03:15:24,094 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/C of 42b167b16e2c28db0ea066b1d94c2c1f into 67d54670e4ee4d19b2d40f166456b051(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:24,095 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:24,095 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/C, priority=13, startTime=1731813323624; duration=0sec 2024-11-17T03:15:24,095 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:24,095 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:C 2024-11-17T03:15:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. as already flushing 2024-11-17T03:15:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813384149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813384150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813384152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813384153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813324142, max timestamp=1731813324151Num uniques 859; 2024-11-17T03:15:24,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:15:24,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813324139, max timestamp=1731813324151Num uniques 859; 2024-11-17T03:15:24,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:15:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813384272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813384273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813384274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813384277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,352 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T03:15:24,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813384377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813384387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813384387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813384397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,469 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/46d6cc447cec4727b37412dcc14d0ce6 2024-11-17T03:15:24,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/a745a2de1322475f8cf8d2f2bd85a53e is 50, key is test_row_0/B:col10/1731813321969/Put/seqid=0 2024-11-17T03:15:24,478 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/9b08f3ee569747e4a6179d89d0711f4a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/9b08f3ee569747e4a6179d89d0711f4a 2024-11-17T03:15:24,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742021_1197 (size=12151) 2024-11-17T03:15:24,484 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/A of 42b167b16e2c28db0ea066b1d94c2c1f into 9b08f3ee569747e4a6179d89d0711f4a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:24,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:24,484 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/a745a2de1322475f8cf8d2f2bd85a53e 2024-11-17T03:15:24,484 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/A, priority=13, startTime=1731813323624; duration=0sec 2024-11-17T03:15:24,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:24,484 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:A 2024-11-17T03:15:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/f2c9dde1cc5e44a1b56df835a12a211a is 50, key is test_row_0/C:col10/1731813321969/Put/seqid=0 2024-11-17T03:15:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742022_1198 (size=12151) 2024-11-17T03:15:24,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813384587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813384607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813384897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,899 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/f2c9dde1cc5e44a1b56df835a12a211a 2024-11-17T03:15:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/46d6cc447cec4727b37412dcc14d0ce6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6 2024-11-17T03:15:24,907 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6, entries=150, sequenceid=235, filesize=30.4 K 2024-11-17T03:15:24,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/a745a2de1322475f8cf8d2f2bd85a53e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/a745a2de1322475f8cf8d2f2bd85a53e 2024-11-17T03:15:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813384907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813384907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:24,913 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/a745a2de1322475f8cf8d2f2bd85a53e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T03:15:24,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/f2c9dde1cc5e44a1b56df835a12a211a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/f2c9dde1cc5e44a1b56df835a12a211a 2024-11-17T03:15:24,917 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/f2c9dde1cc5e44a1b56df835a12a211a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T03:15:24,918 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 42b167b16e2c28db0ea066b1d94c2c1f in 1289ms, sequenceid=235, compaction requested=false 2024-11-17T03:15:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:24,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:24,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T03:15:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T03:15:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:24,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=138.65 KB heapSize=398.55 KB 2024-11-17T03:15:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:24,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:24,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T03:15:24,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9710 sec 2024-11-17T03:15:24,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9750 sec 2024-11-17T03:15:24,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111754a072bd3ab3485a8fb7d5921c286718_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:24,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742023_1199 (size=14994) 2024-11-17T03:15:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813384949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813385066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813385277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:25,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111754a072bd3ab3485a8fb7d5921c286718_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111754a072bd3ab3485a8fb7d5921c286718_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:25,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b118ad137e6840a69b46ad2092f14573, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:25,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b118ad137e6840a69b46ad2092f14573 is 175, key is test_row_0/A:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:25,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742024_1200 (size=39949) 2024-11-17T03:15:25,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813385407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813385417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:25,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813385417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813385587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:25,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b118ad137e6840a69b46ad2092f14573 2024-11-17T03:15:25,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/899c0560e3ff4eaab08c4aeb5916752a is 50, key is test_row_0/B:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:25,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742025_1201 (size=12301) 2024-11-17T03:15:26,034 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-17T03:15:26,034 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:26,034 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,034 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:26,035 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-17T03:15:26,035 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:26,035 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,035 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:26,037 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-17T03:15:26,037 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:26,037 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,037 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:26,043 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-17T03:15:26,043 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:26,043 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,043 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:26,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41534 deadline: 1731813386097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T03:15:26,118 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:26,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/899c0560e3ff4eaab08c4aeb5916752a 2024-11-17T03:15:26,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/165c7841206343b9b7cd400e63241aad is 50, key is test_row_0/C:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:26,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742026_1202 (size=12301) 2024-11-17T03:15:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41490 deadline: 1731813386418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:26,421 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 42b167b16e2c28db0ea066b1d94c2c1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:26,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41518 deadline: 1731813386428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:26,428 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 42b167b16e2c28db0ea066b1d94c2c1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41544 deadline: 1731813386437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:26,438 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 42b167b16e2c28db0ea066b1d94c2c1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:15:26,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/165c7841206343b9b7cd400e63241aad 2024-11-17T03:15:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b118ad137e6840a69b46ad2092f14573 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573 2024-11-17T03:15:26,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573, entries=200, sequenceid=270, filesize=39.0 K 2024-11-17T03:15:26,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/899c0560e3ff4eaab08c4aeb5916752a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/899c0560e3ff4eaab08c4aeb5916752a 2024-11-17T03:15:26,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/899c0560e3ff4eaab08c4aeb5916752a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T03:15:26,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/165c7841206343b9b7cd400e63241aad as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/165c7841206343b9b7cd400e63241aad 2024-11-17T03:15:26,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/165c7841206343b9b7cd400e63241aad, entries=150, sequenceid=270, filesize=12.0 K 2024-11-17T03:15:26,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~398.50 KB/408064, currentSize=40.25 KB/41220 for 42b167b16e2c28db0ea066b1d94c2c1f in 1678ms, sequenceid=270, compaction requested=true 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:26,599 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 42b167b16e2c28db0ea066b1d94c2c1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:26,599 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:26,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:26,600 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:26,600 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:26,600 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/B is initiating minor compaction (all files) 2024-11-17T03:15:26,600 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/A is initiating minor compaction (all files) 2024-11-17T03:15:26,600 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/A in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:26,600 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/B in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:26,600 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/4c89da967f3f42bead9bc0e185a1e5c9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/a745a2de1322475f8cf8d2f2bd85a53e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/899c0560e3ff4eaab08c4aeb5916752a] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=36.3 K 2024-11-17T03:15:26,600 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/9b08f3ee569747e4a6179d89d0711f4a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=100.3 K 2024-11-17T03:15:26,600 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:26,600 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/9b08f3ee569747e4a6179d89d0711f4a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573] 2024-11-17T03:15:26,601 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c89da967f3f42bead9bc0e185a1e5c9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731813321326 2024-11-17T03:15:26,601 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b08f3ee569747e4a6179d89d0711f4a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731813321326 2024-11-17T03:15:26,601 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a745a2de1322475f8cf8d2f2bd85a53e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731813321967 2024-11-17T03:15:26,601 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d6cc447cec4727b37412dcc14d0ce6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731813321967 2024-11-17T03:15:26,601 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899c0560e3ff4eaab08c4aeb5916752a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731813324266 2024-11-17T03:15:26,601 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b118ad137e6840a69b46ad2092f14573, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731813324266 2024-11-17T03:15:26,610 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#B#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:26,611 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:26,611 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/95681ce6c8504cacb08afcfc4d218062 is 50, key is test_row_0/B:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:26,616 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111702be04b3930b48939cb8551d92ace3d5_42b167b16e2c28db0ea066b1d94c2c1f store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:26,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742027_1203 (size=12949) 2024-11-17T03:15:26,637 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111702be04b3930b48939cb8551d92ace3d5_42b167b16e2c28db0ea066b1d94c2c1f, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:26,637 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111702be04b3930b48939cb8551d92ace3d5_42b167b16e2c28db0ea066b1d94c2c1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:26,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742028_1204 (size=4469) 2024-11-17T03:15:26,641 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#A#compaction#169 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:26,642 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/63f1e0bc75d944718f80de6e5df97af4 is 175, key is test_row_0/A:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742029_1205 (size=31903) 2024-11-17T03:15:26,652 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/63f1e0bc75d944718f80de6e5df97af4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/63f1e0bc75d944718f80de6e5df97af4 2024-11-17T03:15:26,657 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/A of 42b167b16e2c28db0ea066b1d94c2c1f into 63f1e0bc75d944718f80de6e5df97af4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:26,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:26,657 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/A, priority=13, startTime=1731813326599; duration=0sec 2024-11-17T03:15:26,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:26,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:A 2024-11-17T03:15:26,657 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:26,658 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:26,658 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 42b167b16e2c28db0ea066b1d94c2c1f/C is initiating minor compaction (all files) 2024-11-17T03:15:26,658 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 42b167b16e2c28db0ea066b1d94c2c1f/C in TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:26,658 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/67d54670e4ee4d19b2d40f166456b051, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/f2c9dde1cc5e44a1b56df835a12a211a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/165c7841206343b9b7cd400e63241aad] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp, totalSize=36.3 K 2024-11-17T03:15:26,659 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d54670e4ee4d19b2d40f166456b051, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731813321326 2024-11-17T03:15:26,659 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c9dde1cc5e44a1b56df835a12a211a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731813321967 2024-11-17T03:15:26,659 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 165c7841206343b9b7cd400e63241aad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731813324266 2024-11-17T03:15:26,671 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 42b167b16e2c28db0ea066b1d94c2c1f#C#compaction#170 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:26,672 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/71bcf8f8f24649b0b1d69fcae43fd743 is 50, key is test_row_0/C:col10/1731813324272/Put/seqid=0 2024-11-17T03:15:26,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742030_1206 (size=12949) 2024-11-17T03:15:27,030 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/95681ce6c8504cacb08afcfc4d218062 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/95681ce6c8504cacb08afcfc4d218062 2024-11-17T03:15:27,037 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/B of 42b167b16e2c28db0ea066b1d94c2c1f into 95681ce6c8504cacb08afcfc4d218062(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:27,037 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:27,037 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/B, priority=13, startTime=1731813326599; duration=0sec 2024-11-17T03:15:27,037 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:27,037 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:B 2024-11-17T03:15:27,087 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/71bcf8f8f24649b0b1d69fcae43fd743 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/71bcf8f8f24649b0b1d69fcae43fd743 2024-11-17T03:15:27,097 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 42b167b16e2c28db0ea066b1d94c2c1f/C of 42b167b16e2c28db0ea066b1d94c2c1f into 71bcf8f8f24649b0b1d69fcae43fd743(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:27,097 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:27,097 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., storeName=42b167b16e2c28db0ea066b1d94c2c1f/C, priority=13, startTime=1731813326599; duration=0sec 2024-11-17T03:15:27,097 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:27,097 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 42b167b16e2c28db0ea066b1d94c2c1f:C 2024-11-17T03:15:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:27,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:15:27,169 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-17T03:15:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:27,169 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:27,169 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:27,170 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:27,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:27,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e380953681ef4fcfa20e1a781d8f4aaf_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813327117/Put/seqid=0 2024-11-17T03:15:27,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742031_1207 (size=12454) 2024-11-17T03:15:27,238 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-17T03:15:27,238 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:27,238 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:27,238 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:27,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:27,584 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:27,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e380953681ef4fcfa20e1a781d8f4aaf_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e380953681ef4fcfa20e1a781d8f4aaf_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:27,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b2561feab49b45cf909cfdf55f213d92, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:27,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b2561feab49b45cf909cfdf55f213d92 is 175, key is test_row_0/A:col10/1731813327117/Put/seqid=0 2024-11-17T03:15:27,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742032_1208 (size=31255) 2024-11-17T03:15:27,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b2561feab49b45cf909cfdf55f213d92 2024-11-17T03:15:28,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/1f919d34145a453eb04d54e0f98e1e5c is 50, key is test_row_0/B:col10/1731813327117/Put/seqid=0 2024-11-17T03:15:28,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742033_1209 (size=12301) 2024-11-17T03:15:28,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/1f919d34145a453eb04d54e0f98e1e5c 2024-11-17T03:15:28,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/4383bf8b787f43cd9c4fa4990a314f13 is 50, key is test_row_0/C:col10/1731813327117/Put/seqid=0 2024-11-17T03:15:28,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742034_1210 (size=12301) 2024-11-17T03:15:28,448 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-17T03:15:28,448 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:28,448 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,448 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-17T03:15:28,448 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,448 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:28,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:28,448 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,449 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,449 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-17T03:15:28,449 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:28,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:28,449 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,449 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3164 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3183 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1334 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3995 rows 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1326 2024-11-17T03:15:28,449 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3975 rows 2024-11-17T03:15:28,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:15:28,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:15:28,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:28,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:28,450 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T03:15:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:28,453 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813328453"}]},"ts":"1731813328453"} 2024-11-17T03:15:28,454 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T03:15:28,454 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T03:15:28,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:15:28,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, UNASSIGN}] 2024-11-17T03:15:28,457 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, UNASSIGN 2024-11-17T03:15:28,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:28,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, UNASSIGN because future has completed 2024-11-17T03:15:28,459 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:15:28,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:28,612 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:28,613 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:15:28,613 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 42b167b16e2c28db0ea066b1d94c2c1f, disabling compactions & flushes 2024-11-17T03:15:28,614 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:28,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/4383bf8b787f43cd9c4fa4990a314f13 2024-11-17T03:15:28,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/b2561feab49b45cf909cfdf55f213d92 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b2561feab49b45cf909cfdf55f213d92 2024-11-17T03:15:28,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b2561feab49b45cf909cfdf55f213d92, entries=150, sequenceid=283, filesize=30.5 K 2024-11-17T03:15:28,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/1f919d34145a453eb04d54e0f98e1e5c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/1f919d34145a453eb04d54e0f98e1e5c 2024-11-17T03:15:28,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/1f919d34145a453eb04d54e0f98e1e5c, entries=150, sequenceid=283, filesize=12.0 K 2024-11-17T03:15:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/4383bf8b787f43cd9c4fa4990a314f13 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4383bf8b787f43cd9c4fa4990a314f13 2024-11-17T03:15:28,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4383bf8b787f43cd9c4fa4990a314f13, entries=150, sequenceid=283, filesize=12.0 K 2024-11-17T03:15:28,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for 42b167b16e2c28db0ea066b1d94c2c1f in 1685ms, sequenceid=283, compaction requested=false 2024-11-17T03:15:28,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 42b167b16e2c28db0ea066b1d94c2c1f: 2024-11-17T03:15:28,854 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:28,854 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:28,854 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. after waiting 0 ms 2024-11-17T03:15:28,854 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:28,854 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 42b167b16e2c28db0ea066b1d94c2c1f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T03:15:28,855 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=A 2024-11-17T03:15:28,855 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:28,855 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=B 2024-11-17T03:15:28,855 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:28,855 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 42b167b16e2c28db0ea066b1d94c2c1f, store=C 2024-11-17T03:15:28,855 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:28,860 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e65d1d49d020408fb5a8b3a744984af4_42b167b16e2c28db0ea066b1d94c2c1f is 50, key is test_row_0/A:col10/1731813328447/Put/seqid=0 2024-11-17T03:15:28,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742035_1211 (size=9914) 2024-11-17T03:15:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:29,264 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:29,267 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e65d1d49d020408fb5a8b3a744984af4_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e65d1d49d020408fb5a8b3a744984af4_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:29,268 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/52967b8b43844fdabea5e1e32f946fa1, store: [table=TestAcidGuarantees family=A region=42b167b16e2c28db0ea066b1d94c2c1f] 2024-11-17T03:15:29,269 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/52967b8b43844fdabea5e1e32f946fa1 is 175, key is test_row_0/A:col10/1731813328447/Put/seqid=0 2024-11-17T03:15:29,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742036_1212 (size=22561) 2024-11-17T03:15:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:29,674 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/52967b8b43844fdabea5e1e32f946fa1 2024-11-17T03:15:29,685 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/624c218fce384426a4707cee5e95048c is 50, key is test_row_0/B:col10/1731813328447/Put/seqid=0 2024-11-17T03:15:29,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742037_1213 (size=9857) 2024-11-17T03:15:30,092 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/624c218fce384426a4707cee5e95048c 2024-11-17T03:15:30,106 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/fa1dcfa2d71b459882ee3dd2bd731252 is 50, key is test_row_0/C:col10/1731813328447/Put/seqid=0 2024-11-17T03:15:30,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742038_1214 (size=9857) 2024-11-17T03:15:30,510 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/fa1dcfa2d71b459882ee3dd2bd731252 2024-11-17T03:15:30,515 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/A/52967b8b43844fdabea5e1e32f946fa1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/52967b8b43844fdabea5e1e32f946fa1 2024-11-17T03:15:30,520 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/52967b8b43844fdabea5e1e32f946fa1, entries=100, sequenceid=290, filesize=22.0 K 2024-11-17T03:15:30,521 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/B/624c218fce384426a4707cee5e95048c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/624c218fce384426a4707cee5e95048c 2024-11-17T03:15:30,526 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/624c218fce384426a4707cee5e95048c, entries=100, sequenceid=290, filesize=9.6 K 2024-11-17T03:15:30,527 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/.tmp/C/fa1dcfa2d71b459882ee3dd2bd731252 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/fa1dcfa2d71b459882ee3dd2bd731252 2024-11-17T03:15:30,531 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/fa1dcfa2d71b459882ee3dd2bd731252, entries=100, sequenceid=290, filesize=9.6 K 2024-11-17T03:15:30,532 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 42b167b16e2c28db0ea066b1d94c2c1f in 1678ms, sequenceid=290, compaction requested=true 2024-11-17T03:15:30,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/21681a6a29b442a2b5e50f1fe4fc96f7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/6127742709fd4d5c8caa50db46e2d3e3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fe023372c5924d198576711a5a04c0fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/3d9c50cd70f74cd39ecbd684cbb1de1f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/9b08f3ee569747e4a6179d89d0711f4a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573] to archive 2024-11-17T03:15:30,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:30,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fc5c58991d4243f9942584d0e3c6efa1 2024-11-17T03:15:30,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/456ad50e120640df96a9bc1ab8de01c0 2024-11-17T03:15:30,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/21681a6a29b442a2b5e50f1fe4fc96f7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/21681a6a29b442a2b5e50f1fe4fc96f7 2024-11-17T03:15:30,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/5cf874877c394cabab2edd2a0e91c1dc 2024-11-17T03:15:30,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/155d7dae42f9469e83181ee94e4335f1 2024-11-17T03:15:30,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/6127742709fd4d5c8caa50db46e2d3e3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/6127742709fd4d5c8caa50db46e2d3e3 2024-11-17T03:15:30,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/7978a1024528411c8e7c927723194c67 2024-11-17T03:15:30,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/0257dda9b0e74f9a87f51361bd000b0e 2024-11-17T03:15:30,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/8e23716d3d8344c99de53e6e3868b8eb 2024-11-17T03:15:30,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fe023372c5924d198576711a5a04c0fd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/fe023372c5924d198576711a5a04c0fd 2024-11-17T03:15:30,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/4cce27b08f804bb6a4995f4a78db1e56 2024-11-17T03:15:30,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/00013ed60a3044b58e59d4090172a977 2024-11-17T03:15:30,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/e3f2c15ffaad466c82d9e9fede8f9f29 2024-11-17T03:15:30,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/3d9c50cd70f74cd39ecbd684cbb1de1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/3d9c50cd70f74cd39ecbd684cbb1de1f 2024-11-17T03:15:30,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/2529d2ff37254a2a87f8d5935a56225a 2024-11-17T03:15:30,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/9b08f3ee569747e4a6179d89d0711f4a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/9b08f3ee569747e4a6179d89d0711f4a 2024-11-17T03:15:30,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/1e9f2835f5d346be870f4db1770da86e 2024-11-17T03:15:30,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/46d6cc447cec4727b37412dcc14d0ce6 2024-11-17T03:15:30,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b118ad137e6840a69b46ad2092f14573 2024-11-17T03:15:30,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d3e11553da0a40d0b1799e8cfdf06016, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/7883c07793054b08a9b76d99a35b223f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/df48b2f468394be8b209970547935aba, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b5832965f8ac48bc967935fdaea1c7d3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/598f64e0af5f4a24a4f9c31e762ef236, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/9fc3131a641d4a8eb9279471d4a14d2a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/aa2a399ea4ec497e899cdd4ff2b0daf8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/de8a53a643ac45dca6f16ba02d129a8f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/c6c8ca6c9422475cb3cecf5c44c8f314, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d2333150c5cb4e01a88ec7d8c9a0580f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/20a8b254e6de4bcdab6436621d2e7fa2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e89187d02cb94a99a3d0fae3b235c57f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b97ac7bb9cb74fde87ba1fd1c359075e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e32aad34a3634c559da631036c4dec71, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/8fd9ccc01cd1433cb3572da56f7683d9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/4c89da967f3f42bead9bc0e185a1e5c9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/402eae963f1f484d874976bb3c993ca9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/a745a2de1322475f8cf8d2f2bd85a53e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/899c0560e3ff4eaab08c4aeb5916752a] to archive 2024-11-17T03:15:30,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:30,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d3e11553da0a40d0b1799e8cfdf06016 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d3e11553da0a40d0b1799e8cfdf06016 2024-11-17T03:15:30,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/7883c07793054b08a9b76d99a35b223f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/7883c07793054b08a9b76d99a35b223f 2024-11-17T03:15:30,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/df48b2f468394be8b209970547935aba to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/df48b2f468394be8b209970547935aba 2024-11-17T03:15:30,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b5832965f8ac48bc967935fdaea1c7d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b5832965f8ac48bc967935fdaea1c7d3 2024-11-17T03:15:30,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/598f64e0af5f4a24a4f9c31e762ef236 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/598f64e0af5f4a24a4f9c31e762ef236 2024-11-17T03:15:30,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/9fc3131a641d4a8eb9279471d4a14d2a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/9fc3131a641d4a8eb9279471d4a14d2a 2024-11-17T03:15:30,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/aa2a399ea4ec497e899cdd4ff2b0daf8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/aa2a399ea4ec497e899cdd4ff2b0daf8 2024-11-17T03:15:30,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/de8a53a643ac45dca6f16ba02d129a8f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/de8a53a643ac45dca6f16ba02d129a8f 2024-11-17T03:15:30,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/c6c8ca6c9422475cb3cecf5c44c8f314 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/c6c8ca6c9422475cb3cecf5c44c8f314 2024-11-17T03:15:30,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d2333150c5cb4e01a88ec7d8c9a0580f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/d2333150c5cb4e01a88ec7d8c9a0580f 2024-11-17T03:15:30,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/20a8b254e6de4bcdab6436621d2e7fa2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/20a8b254e6de4bcdab6436621d2e7fa2 2024-11-17T03:15:30,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e89187d02cb94a99a3d0fae3b235c57f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e89187d02cb94a99a3d0fae3b235c57f 2024-11-17T03:15:30,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b97ac7bb9cb74fde87ba1fd1c359075e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/b97ac7bb9cb74fde87ba1fd1c359075e 2024-11-17T03:15:30,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e32aad34a3634c559da631036c4dec71 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/e32aad34a3634c559da631036c4dec71 2024-11-17T03:15:30,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/8fd9ccc01cd1433cb3572da56f7683d9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/8fd9ccc01cd1433cb3572da56f7683d9 2024-11-17T03:15:30,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/4c89da967f3f42bead9bc0e185a1e5c9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/4c89da967f3f42bead9bc0e185a1e5c9 2024-11-17T03:15:30,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/402eae963f1f484d874976bb3c993ca9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/402eae963f1f484d874976bb3c993ca9 2024-11-17T03:15:30,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/a745a2de1322475f8cf8d2f2bd85a53e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/a745a2de1322475f8cf8d2f2bd85a53e 2024-11-17T03:15:30,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/899c0560e3ff4eaab08c4aeb5916752a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/899c0560e3ff4eaab08c4aeb5916752a 2024-11-17T03:15:30,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b1b3542c43d40d1a38df159fe44706a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/6cd5ca95c285465fac6f2404213ed810, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/96033aa6ca0a47d88eaf57fed03bc3ea, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/b372e3f3947f425d9baa04f781f9ed64, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/19f638379a5148d5bbc12a27fe3c8089, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/5edcdad76d1740ea96525907cdb64d81, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cdaad50ba210426090c8d9d1f0b2ffc1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cafcb883a4fc4623bb128223d51ae7e4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/2ad6beac8f9741d28382fc0d2e2c9111, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/930e21d74ad34bb9a130f63c9bdc540c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/e528b81c8c244b54b2797770a62d2deb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/259ac477a5344e91bd93a67929ced4fa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/1fd465c77cf74058a4adbd7353b92e47, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4c3880ce2ee844b389564a742948283a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/8ec8483e866b4da08594992623c394cd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/67d54670e4ee4d19b2d40f166456b051, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b12e973073446e0b4c91aed1ba65cd4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/f2c9dde1cc5e44a1b56df835a12a211a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/165c7841206343b9b7cd400e63241aad] to archive 2024-11-17T03:15:30,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:30,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b1b3542c43d40d1a38df159fe44706a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b1b3542c43d40d1a38df159fe44706a 2024-11-17T03:15:30,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/6cd5ca95c285465fac6f2404213ed810 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/6cd5ca95c285465fac6f2404213ed810 2024-11-17T03:15:30,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/96033aa6ca0a47d88eaf57fed03bc3ea to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/96033aa6ca0a47d88eaf57fed03bc3ea 2024-11-17T03:15:30,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/b372e3f3947f425d9baa04f781f9ed64 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/b372e3f3947f425d9baa04f781f9ed64 2024-11-17T03:15:30,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/19f638379a5148d5bbc12a27fe3c8089 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/19f638379a5148d5bbc12a27fe3c8089 2024-11-17T03:15:30,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/5edcdad76d1740ea96525907cdb64d81 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/5edcdad76d1740ea96525907cdb64d81 2024-11-17T03:15:30,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cdaad50ba210426090c8d9d1f0b2ffc1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cdaad50ba210426090c8d9d1f0b2ffc1 2024-11-17T03:15:30,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cafcb883a4fc4623bb128223d51ae7e4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/cafcb883a4fc4623bb128223d51ae7e4 2024-11-17T03:15:30,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/2ad6beac8f9741d28382fc0d2e2c9111 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/2ad6beac8f9741d28382fc0d2e2c9111 2024-11-17T03:15:30,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/930e21d74ad34bb9a130f63c9bdc540c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/930e21d74ad34bb9a130f63c9bdc540c 2024-11-17T03:15:30,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/e528b81c8c244b54b2797770a62d2deb to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/e528b81c8c244b54b2797770a62d2deb 2024-11-17T03:15:30,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/259ac477a5344e91bd93a67929ced4fa to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/259ac477a5344e91bd93a67929ced4fa 2024-11-17T03:15:30,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/1fd465c77cf74058a4adbd7353b92e47 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/1fd465c77cf74058a4adbd7353b92e47 2024-11-17T03:15:30,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4c3880ce2ee844b389564a742948283a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4c3880ce2ee844b389564a742948283a 2024-11-17T03:15:30,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/8ec8483e866b4da08594992623c394cd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/8ec8483e866b4da08594992623c394cd 2024-11-17T03:15:30,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/67d54670e4ee4d19b2d40f166456b051 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/67d54670e4ee4d19b2d40f166456b051 2024-11-17T03:15:30,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b12e973073446e0b4c91aed1ba65cd4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/0b12e973073446e0b4c91aed1ba65cd4 2024-11-17T03:15:30,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/f2c9dde1cc5e44a1b56df835a12a211a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/f2c9dde1cc5e44a1b56df835a12a211a 2024-11-17T03:15:30,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/165c7841206343b9b7cd400e63241aad to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/165c7841206343b9b7cd400e63241aad 2024-11-17T03:15:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:30,599 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/recovered.edits/293.seqid, newMaxSeqId=293, maxSeqId=4 2024-11-17T03:15:30,600 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. 2024-11-17T03:15:30,600 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 42b167b16e2c28db0ea066b1d94c2c1f: Waiting for close lock at 1731813328613Running coprocessor pre-close hooks at 1731813328613Disabling compacts and flushes for region at 1731813328613Disabling writes for close at 1731813328854 (+241 ms)Obtaining lock to block concurrent updates at 1731813328854Preparing flush snapshotting stores in 42b167b16e2c28db0ea066b1d94c2c1f at 1731813328854Finished memstore snapshotting TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731813328855 (+1 ms)Flushing stores of TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f. at 1731813328855Flushing 42b167b16e2c28db0ea066b1d94c2c1f/A: creating writer at 1731813328855Flushing mob file 42b167b16e2c28db0ea066b1d94c2c1f/A: appending metadata at 1731813328860 (+5 ms)Flushing mob file 42b167b16e2c28db0ea066b1d94c2c1f/A: closing flushed file at 1731813328860Flushing 42b167b16e2c28db0ea066b1d94c2c1f/A: appending metadata at 1731813329268 (+408 ms)Flushing 42b167b16e2c28db0ea066b1d94c2c1f/A: closing flushed file at 1731813329268Flushing 42b167b16e2c28db0ea066b1d94c2c1f/B: creating writer at 1731813329681 (+413 ms)Flushing 42b167b16e2c28db0ea066b1d94c2c1f/B: appending metadata at 1731813329685 (+4 ms)Flushing 42b167b16e2c28db0ea066b1d94c2c1f/B: closing flushed file at 1731813329685Flushing 42b167b16e2c28db0ea066b1d94c2c1f/C: creating writer at 1731813330102 (+417 ms)Flushing 42b167b16e2c28db0ea066b1d94c2c1f/C: appending metadata at 1731813330105 (+3 ms)Flushing 42b167b16e2c28db0ea066b1d94c2c1f/C: closing flushed file at 1731813330106 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a3a1f0f: reopening flushed file at 1731813330515 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17625a01: reopening flushed file at 1731813330520 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54cd4c1f: reopening flushed file at 1731813330526 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 42b167b16e2c28db0ea066b1d94c2c1f in 1678ms, sequenceid=290, compaction requested=true at 1731813330532 (+6 ms)Writing region close event to WAL at 1731813330595 (+63 ms)Running coprocessor post-close hooks at 1731813330599 (+4 ms)Closed at 1731813330600 (+1 ms) 2024-11-17T03:15:30,602 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:30,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=42b167b16e2c28db0ea066b1d94c2c1f, regionState=CLOSED 2024-11-17T03:15:30,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:30,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T03:15:30,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 42b167b16e2c28db0ea066b1d94c2c1f, server=695c90f4bb18,38753,1731813266381 in 2.1450 sec 2024-11-17T03:15:30,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-17T03:15:30,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=42b167b16e2c28db0ea066b1d94c2c1f, UNASSIGN in 2.1500 sec 2024-11-17T03:15:30,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T03:15:30,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1520 sec 2024-11-17T03:15:30,609 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813330609"}]},"ts":"1731813330609"} 2024-11-17T03:15:30,610 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T03:15:30,610 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T03:15:30,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1610 sec 2024-11-17T03:15:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T03:15:32,617 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:32,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T03:15:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,619 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T03:15:32,620 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/recovered.edits] 2024-11-17T03:15:32,628 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/52967b8b43844fdabea5e1e32f946fa1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/52967b8b43844fdabea5e1e32f946fa1 2024-11-17T03:15:32,629 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/63f1e0bc75d944718f80de6e5df97af4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/63f1e0bc75d944718f80de6e5df97af4 2024-11-17T03:15:32,630 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b2561feab49b45cf909cfdf55f213d92 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/A/b2561feab49b45cf909cfdf55f213d92 2024-11-17T03:15:32,632 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/1f919d34145a453eb04d54e0f98e1e5c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/1f919d34145a453eb04d54e0f98e1e5c 2024-11-17T03:15:32,633 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/624c218fce384426a4707cee5e95048c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/624c218fce384426a4707cee5e95048c 2024-11-17T03:15:32,635 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/95681ce6c8504cacb08afcfc4d218062 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/B/95681ce6c8504cacb08afcfc4d218062 2024-11-17T03:15:32,637 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4383bf8b787f43cd9c4fa4990a314f13 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/4383bf8b787f43cd9c4fa4990a314f13 2024-11-17T03:15:32,638 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/71bcf8f8f24649b0b1d69fcae43fd743 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/71bcf8f8f24649b0b1d69fcae43fd743 2024-11-17T03:15:32,639 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/fa1dcfa2d71b459882ee3dd2bd731252 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/C/fa1dcfa2d71b459882ee3dd2bd731252 2024-11-17T03:15:32,642 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/recovered.edits/293.seqid to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f/recovered.edits/293.seqid 2024-11-17T03:15:32,642 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,642 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T03:15:32,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T03:15:32,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T03:15:32,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111733e6271054594a4f836ee714f68651ae_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111733e6271054594a4f836ee714f68651ae_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174ece741bf4a845c08c5c67cd20db3fda_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174ece741bf4a845c08c5c67cd20db3fda_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111754a072bd3ab3485a8fb7d5921c286718_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111754a072bd3ab3485a8fb7d5921c286718_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111766e638e9c0bf40c6aee2d93bc74dfe08_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111766e638e9c0bf40c6aee2d93bc74dfe08_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,652 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f7f521d6db946819a01f2491b19abf8_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f7f521d6db946819a01f2491b19abf8_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111791d0fba1289a42f1881f7737ca3030be_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111791d0fba1289a42f1881f7737ca3030be_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a87104650f8746eda055f397ea7d4cd3_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a87104650f8746eda055f397ea7d4cd3_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,655 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c1865df5e09243e88820eca68f41471d_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c1865df5e09243e88820eca68f41471d_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,657 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c32d256a13ae450e8e42e8c5ac5cf7bf_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c32d256a13ae450e8e42e8c5ac5cf7bf_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,658 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c78fdced7d3046f5aa855ed257c97f87_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c78fdced7d3046f5aa855ed257c97f87_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,659 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d2cffe4823c2417b8ef25eba7b4061ce_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d2cffe4823c2417b8ef25eba7b4061ce_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,660 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e380953681ef4fcfa20e1a781d8f4aaf_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e380953681ef4fcfa20e1a781d8f4aaf_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,661 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e65d1d49d020408fb5a8b3a744984af4_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e65d1d49d020408fb5a8b3a744984af4_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,663 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f6e6a1a3315f4381a13ab91a45aed535_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f6e6a1a3315f4381a13ab91a45aed535_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,664 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f8ead41ba9ff46ff9afbba28c93d9a26_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f8ead41ba9ff46ff9afbba28c93d9a26_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,665 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fc5ab00a0ab04959b2c393f0bded3fb2_42b167b16e2c28db0ea066b1d94c2c1f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fc5ab00a0ab04959b2c393f0bded3fb2_42b167b16e2c28db0ea066b1d94c2c1f 2024-11-17T03:15:32,666 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T03:15:32,668 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,670 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T03:15:32,672 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T03:15:32,674 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,674 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T03:15:32,674 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731813332674"}]},"ts":"9223372036854775807"} 2024-11-17T03:15:32,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T03:15:32,677 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 42b167b16e2c28db0ea066b1d94c2c1f, NAME => 'TestAcidGuarantees,,1731813302848.42b167b16e2c28db0ea066b1d94c2c1f.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T03:15:32,677 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T03:15:32,677 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731813332677"}]},"ts":"9223372036854775807"} 2024-11-17T03:15:32,679 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T03:15:32,680 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 62 msec 2024-11-17T03:15:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T03:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T03:15:32,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:32,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 247) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_570995009_22 at /127.0.0.1:36022 [Waiting for operation #273] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-131647061_22 at /127.0.0.1:44324 [Waiting for operation #212] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-131647061_22 at /127.0.0.1:44280 [Waiting for operation #219] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 384) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2574 (was 1271) - AvailableMemoryMB LEAK? - 2024-11-17T03:15:32,758 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=2569 2024-11-17T03:15:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:15:32,760 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:15:32,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:32,763 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T03:15:32,763 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:32,763 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-17T03:15:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T03:15:32,766 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T03:15:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742039_1215 (size=1061) 2024-11-17T03:15:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T03:15:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T03:15:33,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:15:33,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742040_1216 (size=53) 2024-11-17T03:15:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T03:15:33,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:33,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a5d5a03df69af3fa2659ea9597853e20, disabling compactions & flushes 2024-11-17T03:15:33,595 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. after waiting 0 ms 2024-11-17T03:15:33,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,595 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,595 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5d5a03df69af3fa2659ea9597853e20: Waiting for close lock at 1731813333595Disabling compacts and flushes for region at 1731813333595Disabling writes for close at 1731813333595Writing region close event to WAL at 1731813333595Closed at 1731813333595 2024-11-17T03:15:33,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T03:15:33,597 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731813333596"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731813333596"}]},"ts":"1731813333596"} 2024-11-17T03:15:33,598 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T03:15:33,599 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T03:15:33,599 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813333599"}]},"ts":"1731813333599"} 2024-11-17T03:15:33,601 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T03:15:33,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, ASSIGN}] 2024-11-17T03:15:33,602 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, ASSIGN 2024-11-17T03:15:33,603 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, ASSIGN; state=OFFLINE, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=false 2024-11-17T03:15:33,754 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a5d5a03df69af3fa2659ea9597853e20, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, ASSIGN because future has completed 2024-11-17T03:15:33,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T03:15:33,911 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,911 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:15:33,912 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,912 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:33,912 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,912 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,913 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,914 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:33,915 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5d5a03df69af3fa2659ea9597853e20 columnFamilyName A 2024-11-17T03:15:33,915 DEBUG [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:33,915 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.HStore(327): Store=a5d5a03df69af3fa2659ea9597853e20/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:33,915 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,917 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:33,917 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5d5a03df69af3fa2659ea9597853e20 columnFamilyName B 2024-11-17T03:15:33,917 DEBUG [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:33,923 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.HStore(327): Store=a5d5a03df69af3fa2659ea9597853e20/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:33,923 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,924 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:33,924 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5d5a03df69af3fa2659ea9597853e20 columnFamilyName C 2024-11-17T03:15:33,924 DEBUG [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:33,925 INFO [StoreOpener-a5d5a03df69af3fa2659ea9597853e20-1 {}] regionserver.HStore(327): Store=a5d5a03df69af3fa2659ea9597853e20/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:33,925 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,925 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,926 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,926 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,927 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,927 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,928 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:15:33,929 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,931 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:15:33,932 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened a5d5a03df69af3fa2659ea9597853e20; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59599696, jitterRate=-0.11189532279968262}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:15:33,932 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:33,933 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for a5d5a03df69af3fa2659ea9597853e20: Running coprocessor pre-open hook at 1731813333912Writing region info on filesystem at 1731813333912Initializing all the Stores at 1731813333913 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813333913Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813333913Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813333913Cleaning up temporary data from old regions at 1731813333927 (+14 ms)Running coprocessor post-open hooks at 1731813333932 (+5 ms)Region opened successfully at 1731813333933 (+1 ms) 2024-11-17T03:15:33,934 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., pid=61, masterSystemTime=1731813333907 2024-11-17T03:15:33,936 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,936 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:33,936 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a5d5a03df69af3fa2659ea9597853e20, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:33,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:33,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T03:15:33,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 in 182 msec 2024-11-17T03:15:33,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-17T03:15:33,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, ASSIGN in 339 msec 2024-11-17T03:15:33,943 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T03:15:33,943 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813333943"}]},"ts":"1731813333943"} 2024-11-17T03:15:33,945 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T03:15:33,946 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T03:15:33,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-17T03:15:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T03:15:34,917 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:34,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533c1a83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,919 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd48bcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423dbe50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:15:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:15:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c91a04a 2024-11-17T03:15:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:15:34,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:15:34,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2dccd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba5a890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333018c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661204ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52812d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3e1a96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40655b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c4c450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18495ae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d355d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26414732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,940 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0145c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf4401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,943 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5721bf6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,944 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6adfbca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52267517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67304b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368adf07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4b43fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a8602b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,952 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58efcf70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa78cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,954 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358beaa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,955 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa7e9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06ee61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a55e063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4790b32d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a31e52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:15:34,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:15:34,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:15:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:15:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:15:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734ce248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:15:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:15:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:34,962 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:15:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7fe08b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:15:34,963 DEBUG [Thread-1042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,963 DEBUG [Thread-1027 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,963 DEBUG [Thread-1031 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,963 DEBUG [Thread-1034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,963 DEBUG [Thread-1025 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,964 DEBUG [Thread-1042 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,964 DEBUG [Thread-1042 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,964 DEBUG [Thread-1036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,968 DEBUG [Thread-1038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,981 DEBUG [Thread-1038 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:34,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:15:34,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,982 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:34,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,991 DEBUG [Thread-1029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:34,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:34,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-17T03:15:34,996 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:34,997 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:34,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T03:15:35,004 DEBUG [Thread-1023 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:15:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:15:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:15:35,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:15:35,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b64216f0c9bc4bdc83c443ce2c8d3039 is 50, key is test_row_0/A:col10/1731813334980/Put/seqid=0 2024-11-17T03:15:35,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:15:35,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742041_1217 (size=12001) 2024-11-17T03:15:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813335053, max timestamp=1731813335067Num uniques 859; 2024-11-17T03:15:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:15:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813335053, max timestamp=1731813335067Num uniques 859; 2024-11-17T03:15:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:15:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813335053, max timestamp=1731813335067Num uniques 859; 2024-11-17T03:15:35,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:15:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813395089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813395090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813395092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813395093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813395098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T03:15:35,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813395207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T03:15:35,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b64216f0c9bc4bdc83c443ce2c8d3039 2024-11-17T03:15:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813395417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813395417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813395418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813395418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813395418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:35,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/b43abf8653e845f1b12a7e4652426742 is 50, key is test_row_0/B:col10/1731813334980/Put/seqid=0 2024-11-17T03:15:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742042_1218 (size=12001) 2024-11-17T03:15:35,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/b43abf8653e845f1b12a7e4652426742 2024-11-17T03:15:35,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/73c2cbf8c7144a108a59a44c8ffb8e74 is 50, key is test_row_0/C:col10/1731813334980/Put/seqid=0 2024-11-17T03:15:35,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742043_1219 (size=12001) 2024-11-17T03:15:35,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:35,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T03:15:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813395727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813395728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813395728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813395728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813395728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:35,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:35,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:35,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:35,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/73c2cbf8c7144a108a59a44c8ffb8e74 2024-11-17T03:15:35,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b64216f0c9bc4bdc83c443ce2c8d3039 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b64216f0c9bc4bdc83c443ce2c8d3039 2024-11-17T03:15:35,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b64216f0c9bc4bdc83c443ce2c8d3039, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T03:15:35,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/b43abf8653e845f1b12a7e4652426742 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/b43abf8653e845f1b12a7e4652426742 2024-11-17T03:15:35,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/b43abf8653e845f1b12a7e4652426742, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T03:15:35,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/73c2cbf8c7144a108a59a44c8ffb8e74 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c2cbf8c7144a108a59a44c8ffb8e74 2024-11-17T03:15:35,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c2cbf8c7144a108a59a44c8ffb8e74, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T03:15:35,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a5d5a03df69af3fa2659ea9597853e20 in 1005ms, sequenceid=11, compaction requested=false 2024-11-17T03:15:35,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:36,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T03:15:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:36,071 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T03:15:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:36,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:36,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:36,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:36,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/05ed1e9761994b03bd2219ef14927759 is 50, key is test_row_0/A:col10/1731813335095/Put/seqid=0 2024-11-17T03:15:36,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742044_1220 (size=12001) 2024-11-17T03:15:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T03:15:36,141 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T03:15:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:36,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813396242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813396243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813396243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813396244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813396248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813396356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813396357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813396357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813396357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813396357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,508 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/05ed1e9761994b03bd2219ef14927759 2024-11-17T03:15:36,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/3e020b90ecb04a08ad9cc8ea2b328f8a is 50, key is test_row_0/B:col10/1731813335095/Put/seqid=0 2024-11-17T03:15:36,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742045_1221 (size=12001) 2024-11-17T03:15:36,520 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/3e020b90ecb04a08ad9cc8ea2b328f8a 2024-11-17T03:15:36,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/dc8b028ba26c40d09724ba132ba8e88f is 50, key is test_row_0/C:col10/1731813335095/Put/seqid=0 2024-11-17T03:15:36,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742046_1222 (size=12001) 2024-11-17T03:15:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813396567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813396567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813396568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813396568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813396568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813396877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813396877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813396877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813396878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813396878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:36,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:36,942 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/dc8b028ba26c40d09724ba132ba8e88f 2024-11-17T03:15:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/05ed1e9761994b03bd2219ef14927759 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/05ed1e9761994b03bd2219ef14927759 2024-11-17T03:15:36,957 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/05ed1e9761994b03bd2219ef14927759, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T03:15:36,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/3e020b90ecb04a08ad9cc8ea2b328f8a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/3e020b90ecb04a08ad9cc8ea2b328f8a 2024-11-17T03:15:36,963 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/3e020b90ecb04a08ad9cc8ea2b328f8a, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T03:15:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/dc8b028ba26c40d09724ba132ba8e88f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/dc8b028ba26c40d09724ba132ba8e88f 2024-11-17T03:15:36,983 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/dc8b028ba26c40d09724ba132ba8e88f, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T03:15:36,986 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a5d5a03df69af3fa2659ea9597853e20 in 915ms, sequenceid=47, compaction requested=false 2024-11-17T03:15:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-17T03:15:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-17T03:15:36,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T03:15:36,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-11-17T03:15:37,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0020 sec 2024-11-17T03:15:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T03:15:37,147 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:37,148 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-17T03:15:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T03:15:37,150 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:37,151 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:37,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T03:15:37,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T03:15:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:37,304 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:15:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/5502a4e1dbcf459dabcb570216837359 is 50, key is test_row_0/A:col10/1731813336247/Put/seqid=0 2024-11-17T03:15:37,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742047_1223 (size=12001) 2024-11-17T03:15:37,313 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/5502a4e1dbcf459dabcb570216837359 2024-11-17T03:15:37,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/51d2594fd569432e8bf9e73ecfd840b0 is 50, key is test_row_0/B:col10/1731813336247/Put/seqid=0 2024-11-17T03:15:37,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742048_1224 (size=12001) 2024-11-17T03:15:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:37,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:37,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813337430, max timestamp=1731813337435Num uniques 859; 2024-11-17T03:15:37,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813337430, max timestamp=1731813337435Num uniques 859; 2024-11-17T03:15:37,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813337430, max timestamp=1731813337435Num uniques 859; 2024-11-17T03:15:37,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:15:37,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:15:37,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:15:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T03:15:37,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813397483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813397513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813397513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813397513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813397514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813397627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813397627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813397628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813397628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813397628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,736 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/51d2594fd569432e8bf9e73ecfd840b0 2024-11-17T03:15:37,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/540832c5e5904ac4901ac52a795bd6c8 is 50, key is test_row_0/C:col10/1731813336247/Put/seqid=0 2024-11-17T03:15:37,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742049_1225 (size=12001) 2024-11-17T03:15:37,760 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/540832c5e5904ac4901ac52a795bd6c8 2024-11-17T03:15:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/5502a4e1dbcf459dabcb570216837359 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/5502a4e1dbcf459dabcb570216837359 2024-11-17T03:15:37,774 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/5502a4e1dbcf459dabcb570216837359, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T03:15:37,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/51d2594fd569432e8bf9e73ecfd840b0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/51d2594fd569432e8bf9e73ecfd840b0 2024-11-17T03:15:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T03:15:37,779 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/51d2594fd569432e8bf9e73ecfd840b0, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T03:15:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/540832c5e5904ac4901ac52a795bd6c8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/540832c5e5904ac4901ac52a795bd6c8 2024-11-17T03:15:37,784 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/540832c5e5904ac4901ac52a795bd6c8, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T03:15:37,785 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for a5d5a03df69af3fa2659ea9597853e20 in 481ms, sequenceid=57, compaction requested=true 2024-11-17T03:15:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-17T03:15:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-17T03:15:37,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T03:15:37,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 636 msec 2024-11-17T03:15:37,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 641 msec 2024-11-17T03:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:37,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-17T03:15:37,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:37,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b1d9e588e27241db880c9c73e15c53e6 is 50, key is test_row_0/A:col10/1731813337512/Put/seqid=0 2024-11-17T03:15:37,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813397843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813397844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813397844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813397847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742050_1226 (size=12001) 2024-11-17T03:15:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813397849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b1d9e588e27241db880c9c73e15c53e6 2024-11-17T03:15:37,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4fba7169049643a3a5dd4a1ab0ff6ee9 is 50, key is test_row_0/B:col10/1731813337512/Put/seqid=0 2024-11-17T03:15:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742051_1227 (size=12001) 2024-11-17T03:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813397957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813397958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813397958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813397966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813397966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813398167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813398167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813398167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813398177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813398177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4fba7169049643a3a5dd4a1ab0ff6ee9 2024-11-17T03:15:38,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/de9b8c1aa14b4e9d82d72382e8a7ccf9 is 50, key is test_row_0/C:col10/1731813337512/Put/seqid=0 2024-11-17T03:15:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742052_1228 (size=12001) 2024-11-17T03:15:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T03:15:38,287 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:38,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-17T03:15:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T03:15:38,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:38,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:38,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T03:15:38,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T03:15:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813398476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813398477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813398477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813398487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813398497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:38,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T03:15:38,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:38,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:38,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T03:15:38,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/de9b8c1aa14b4e9d82d72382e8a7ccf9 2024-11-17T03:15:38,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b1d9e588e27241db880c9c73e15c53e6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b1d9e588e27241db880c9c73e15c53e6 2024-11-17T03:15:38,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b1d9e588e27241db880c9c73e15c53e6, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T03:15:38,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4fba7169049643a3a5dd4a1ab0ff6ee9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4fba7169049643a3a5dd4a1ab0ff6ee9 2024-11-17T03:15:38,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4fba7169049643a3a5dd4a1ab0ff6ee9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T03:15:38,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/de9b8c1aa14b4e9d82d72382e8a7ccf9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/de9b8c1aa14b4e9d82d72382e8a7ccf9 2024-11-17T03:15:38,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/de9b8c1aa14b4e9d82d72382e8a7ccf9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-17T03:15:38,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for a5d5a03df69af3fa2659ea9597853e20 in 877ms, sequenceid=94, compaction requested=true 2024-11-17T03:15:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:38,717 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:38,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:38,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:38,718 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:38,719 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:38,719 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:38,720 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,720 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b64216f0c9bc4bdc83c443ce2c8d3039, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/05ed1e9761994b03bd2219ef14927759, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/5502a4e1dbcf459dabcb570216837359, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b1d9e588e27241db880c9c73e15c53e6] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=46.9 K 2024-11-17T03:15:38,721 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b64216f0c9bc4bdc83c443ce2c8d3039, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731813334973 2024-11-17T03:15:38,721 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:38,721 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:38,721 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,721 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/b43abf8653e845f1b12a7e4652426742, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/3e020b90ecb04a08ad9cc8ea2b328f8a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/51d2594fd569432e8bf9e73ecfd840b0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4fba7169049643a3a5dd4a1ab0ff6ee9] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=46.9 K 2024-11-17T03:15:38,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:38,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:38,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:38,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:38,722 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ed1e9761994b03bd2219ef14927759, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813335077 2024-11-17T03:15:38,722 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b43abf8653e845f1b12a7e4652426742, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731813334973 2024-11-17T03:15:38,722 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5502a4e1dbcf459dabcb570216837359, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731813336239 2024-11-17T03:15:38,722 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e020b90ecb04a08ad9cc8ea2b328f8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813335077 2024-11-17T03:15:38,722 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d9e588e27241db880c9c73e15c53e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731813337485 2024-11-17T03:15:38,722 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d2594fd569432e8bf9e73ecfd840b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731813336239 2024-11-17T03:15:38,723 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fba7169049643a3a5dd4a1ab0ff6ee9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731813337485 2024-11-17T03:15:38,737 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#189 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:38,737 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/87d074227341471292b3bf497ace0502 is 50, key is test_row_0/B:col10/1731813337512/Put/seqid=0 2024-11-17T03:15:38,741 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#190 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:38,741 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/6ac99f08825d4494b4f999dbaa7d8307 is 50, key is test_row_0/A:col10/1731813337512/Put/seqid=0 2024-11-17T03:15:38,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742053_1229 (size=12139) 2024-11-17T03:15:38,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T03:15:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,753 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T03:15:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/4177082470654b1189f05f79ab427946 is 50, key is test_row_0/A:col10/1731813337847/Put/seqid=0 2024-11-17T03:15:38,764 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/87d074227341471292b3bf497ace0502 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/87d074227341471292b3bf497ace0502 2024-11-17T03:15:38,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742054_1230 (size=12139) 2024-11-17T03:15:38,772 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into 87d074227341471292b3bf497ace0502(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:38,772 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:38,772 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=12, startTime=1731813338718; duration=0sec 2024-11-17T03:15:38,772 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:38,772 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:38,773 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:38,776 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:38,776 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:38,776 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:38,776 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c2cbf8c7144a108a59a44c8ffb8e74, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/dc8b028ba26c40d09724ba132ba8e88f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/540832c5e5904ac4901ac52a795bd6c8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/de9b8c1aa14b4e9d82d72382e8a7ccf9] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=46.9 K 2024-11-17T03:15:38,776 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c2cbf8c7144a108a59a44c8ffb8e74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731813334973 2024-11-17T03:15:38,778 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting dc8b028ba26c40d09724ba132ba8e88f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731813335077 2024-11-17T03:15:38,778 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 540832c5e5904ac4901ac52a795bd6c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731813336239 2024-11-17T03:15:38,779 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting de9b8c1aa14b4e9d82d72382e8a7ccf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731813337485 2024-11-17T03:15:38,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742055_1231 (size=12001) 2024-11-17T03:15:38,795 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/4177082470654b1189f05f79ab427946 2024-11-17T03:15:38,800 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:38,801 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/1aa380a16c8647dbb2741ebc95571125 is 50, key is test_row_0/C:col10/1731813337512/Put/seqid=0 2024-11-17T03:15:38,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/9820e21f8d034324bf0cc56364bb6d6c is 50, key is test_row_0/B:col10/1731813337847/Put/seqid=0 2024-11-17T03:15:38,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742056_1232 (size=12139) 2024-11-17T03:15:38,846 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/1aa380a16c8647dbb2741ebc95571125 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/1aa380a16c8647dbb2741ebc95571125 2024-11-17T03:15:38,853 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into 1aa380a16c8647dbb2741ebc95571125(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:38,853 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:38,853 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=12, startTime=1731813338721; duration=0sec 2024-11-17T03:15:38,853 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:38,853 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:38,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742057_1233 (size=12001) 2024-11-17T03:15:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T03:15:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:38,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:39,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813339005, max timestamp=1731813339012Num uniques 859; 2024-11-17T03:15:39,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:15:39,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813339005, max timestamp=1731813339012Num uniques 859; 2024-11-17T03:15:39,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813339005, max timestamp=1731813339012Num uniques 859; 2024-11-17T03:15:39,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:15:39,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:15:39,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813399050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813399051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813399053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813399054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813399056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,061 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813399168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813399168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813399168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813399168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813399169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,171 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/6ac99f08825d4494b4f999dbaa7d8307 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/6ac99f08825d4494b4f999dbaa7d8307 2024-11-17T03:15:39,177 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into 6ac99f08825d4494b4f999dbaa7d8307(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:39,177 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:39,177 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=12, startTime=1731813338717; duration=0sec 2024-11-17T03:15:39,177 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:39,177 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:39,274 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/9820e21f8d034324bf0cc56364bb6d6c 2024-11-17T03:15:39,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/b6b972c687cf434d80d831a58e03befa is 50, key is test_row_0/C:col10/1731813337847/Put/seqid=0 2024-11-17T03:15:39,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742058_1234 (size=12001) 2024-11-17T03:15:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813399377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813399377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813399377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813399378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813399378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T03:15:39,687 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/b6b972c687cf434d80d831a58e03befa 2024-11-17T03:15:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813399687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813399687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813399687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813399687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813399687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/4177082470654b1189f05f79ab427946 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/4177082470654b1189f05f79ab427946 2024-11-17T03:15:39,701 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/4177082470654b1189f05f79ab427946, entries=150, sequenceid=103, filesize=11.7 K 2024-11-17T03:15:39,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/9820e21f8d034324bf0cc56364bb6d6c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/9820e21f8d034324bf0cc56364bb6d6c 2024-11-17T03:15:39,707 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/9820e21f8d034324bf0cc56364bb6d6c, entries=150, sequenceid=103, filesize=11.7 K 2024-11-17T03:15:39,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/b6b972c687cf434d80d831a58e03befa as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/b6b972c687cf434d80d831a58e03befa 2024-11-17T03:15:39,714 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/b6b972c687cf434d80d831a58e03befa, entries=150, sequenceid=103, filesize=11.7 K 2024-11-17T03:15:39,715 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for a5d5a03df69af3fa2659ea9597853e20 in 962ms, sequenceid=103, compaction requested=false 2024-11-17T03:15:39,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:39,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:39,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-17T03:15:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-17T03:15:39,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T03:15:39,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-17T03:15:39,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4320 sec 2024-11-17T03:15:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:40,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-17T03:15:40,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:40,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:15:40,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:40,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:15:40,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:40,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:15:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813400203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813400207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/904aa740549040ee91101201a23045e5 is 50, key is test_row_0/A:col10/1731813339053/Put/seqid=0 2024-11-17T03:15:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813400208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813400209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813400210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742059_1235 (size=12151) 2024-11-17T03:15:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813400317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813400327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813400327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813400331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T03:15:40,437 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:40,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T03:15:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T03:15:40,439 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:40,440 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:40,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:40,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813400527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813400537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813400537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T03:15:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813400546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T03:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:40,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/904aa740549040ee91101201a23045e5 2024-11-17T03:15:40,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2b200a29d8794ea7882b50950e0719ae is 50, key is test_row_0/B:col10/1731813339053/Put/seqid=0 2024-11-17T03:15:40,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742060_1236 (size=12151) 2024-11-17T03:15:40,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T03:15:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:40,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T03:15:40,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813400837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813400847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813400847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813400857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:40,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:40,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T03:15:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:40,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:41,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2b200a29d8794ea7882b50950e0719ae 2024-11-17T03:15:41,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4fea6e36ea37482f8a62b77be8f76258 is 50, key is test_row_0/C:col10/1731813339053/Put/seqid=0 2024-11-17T03:15:41,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T03:15:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:41,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742061_1237 (size=12151) 2024-11-17T03:15:41,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4fea6e36ea37482f8a62b77be8f76258 2024-11-17T03:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T03:15:41,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/904aa740549040ee91101201a23045e5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/904aa740549040ee91101201a23045e5 2024-11-17T03:15:41,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/904aa740549040ee91101201a23045e5, entries=150, sequenceid=145, filesize=11.9 K 2024-11-17T03:15:41,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2b200a29d8794ea7882b50950e0719ae as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2b200a29d8794ea7882b50950e0719ae 2024-11-17T03:15:41,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2b200a29d8794ea7882b50950e0719ae, entries=150, sequenceid=145, filesize=11.9 K 2024-11-17T03:15:41,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4fea6e36ea37482f8a62b77be8f76258 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4fea6e36ea37482f8a62b77be8f76258 2024-11-17T03:15:41,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4fea6e36ea37482f8a62b77be8f76258, entries=150, sequenceid=145, filesize=11.9 K 2024-11-17T03:15:41,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~455.95 KB/466896, currentSize=20.13 KB/20610 for a5d5a03df69af3fa2659ea9597853e20 in 891ms, sequenceid=145, compaction requested=true 2024-11-17T03:15:41,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:41,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:41,092 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:41,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:41,092 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:41,093 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:41,094 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:41,094 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,094 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/6ac99f08825d4494b4f999dbaa7d8307, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/4177082470654b1189f05f79ab427946, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/904aa740549040ee91101201a23045e5] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=35.4 K 2024-11-17T03:15:41,093 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:41,094 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:41,094 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,094 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/87d074227341471292b3bf497ace0502, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/9820e21f8d034324bf0cc56364bb6d6c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2b200a29d8794ea7882b50950e0719ae] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=35.4 K 2024-11-17T03:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:41,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:41,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:41,095 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 87d074227341471292b3bf497ace0502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731813337485 2024-11-17T03:15:41,095 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ac99f08825d4494b4f999dbaa7d8307, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731813337485 2024-11-17T03:15:41,096 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4177082470654b1189f05f79ab427946, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731813337843 2024-11-17T03:15:41,096 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9820e21f8d034324bf0cc56364bb6d6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731813337843 2024-11-17T03:15:41,096 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b200a29d8794ea7882b50950e0719ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731813339053 2024-11-17T03:15:41,096 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 904aa740549040ee91101201a23045e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731813339053 2024-11-17T03:15:41,118 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:41,118 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4190b1df2e7c4b27886d155134c8ed1b is 50, key is test_row_0/B:col10/1731813339053/Put/seqid=0 2024-11-17T03:15:41,126 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:41,127 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/355f3254157742c6a79fe21a2cd8beb1 is 50, key is test_row_0/A:col10/1731813339053/Put/seqid=0 2024-11-17T03:15:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742063_1239 (size=12391) 2024-11-17T03:15:41,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742062_1238 (size=12391) 2024-11-17T03:15:41,150 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4190b1df2e7c4b27886d155134c8ed1b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4190b1df2e7c4b27886d155134c8ed1b 2024-11-17T03:15:41,155 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into 4190b1df2e7c4b27886d155134c8ed1b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:41,155 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:41,156 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=13, startTime=1731813341092; duration=0sec 2024-11-17T03:15:41,156 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:41,156 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:41,156 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:41,157 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:41,157 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:41,157 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,157 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/1aa380a16c8647dbb2741ebc95571125, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/b6b972c687cf434d80d831a58e03befa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4fea6e36ea37482f8a62b77be8f76258] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=35.4 K 2024-11-17T03:15:41,157 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aa380a16c8647dbb2741ebc95571125, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731813337485 2024-11-17T03:15:41,158 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b972c687cf434d80d831a58e03befa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731813337843 2024-11-17T03:15:41,158 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fea6e36ea37482f8a62b77be8f76258, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731813339053 2024-11-17T03:15:41,167 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:41,167 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/07dfdf0ef439434eadb164d8be066e17 is 50, key is test_row_0/C:col10/1731813339053/Put/seqid=0 2024-11-17T03:15:41,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742064_1240 (size=12391) 2024-11-17T03:15:41,190 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/07dfdf0ef439434eadb164d8be066e17 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/07dfdf0ef439434eadb164d8be066e17 2024-11-17T03:15:41,198 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into 07dfdf0ef439434eadb164d8be066e17(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:41,198 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:41,198 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=13, startTime=1731813341094; duration=0sec 2024-11-17T03:15:41,198 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:41,198 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:41,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T03:15:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,203 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-17T03:15:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:41,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/cc7454b21db249f7ad405135f59c2db0 is 50, key is test_row_0/A:col10/1731813340208/Put/seqid=0 2024-11-17T03:15:41,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742065_1241 (size=9757) 2024-11-17T03:15:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:41,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813341271, max timestamp=1731813341299Num uniques 859; 2024-11-17T03:15:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:15:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813341271, max timestamp=1731813341299Num uniques 859; 2024-11-17T03:15:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813341271, max timestamp=1731813341299Num uniques 859; 2024-11-17T03:15:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:15:41,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:15:41,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731813341327, max timestamp=1731813341360Num uniques 1018; 2024-11-17T03:15:41,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731813341327, max timestamp=1731813341360Num uniques 1018; 2024-11-17T03:15:41,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T03:15:41,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T03:15:41,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731813341327, max timestamp=1731813341360Num uniques 1018; 2024-11-17T03:15:41,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T03:15:41,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8095, totalHeapSize=18648, min timestamp=1731813341381, max timestamp=1731813341386Num uniques 1127; 2024-11-17T03:15:41,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51605, new segment data size=8095, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-17T03:15:41,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813341381, max timestamp=1731813341385Num uniques 1127; 2024-11-17T03:15:41,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-17T03:15:41,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8094, totalHeapSize=18648, min timestamp=1731813341381, max timestamp=1731813341386Num uniques 1127; 2024-11-17T03:15:41,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51604, new segment data size=8094, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-17T03:15:41,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1731813341401, max timestamp=1731813341411Num uniques 1036; 2024-11-17T03:15:41,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1731813341401, max timestamp=1731813341411Num uniques 1036; 2024-11-17T03:15:41,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T03:15:41,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T03:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1731813341401, max timestamp=1731813341411Num uniques 1036; 2024-11-17T03:15:41,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T03:15:41,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731813341425, max timestamp=1731813341440Num uniques 1045; 2024-11-17T03:15:41,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T03:15:41,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731813341425, max timestamp=1731813341440Num uniques 1045; 2024-11-17T03:15:41,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T03:15:41,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731813341425, max timestamp=1731813341440Num uniques 1045; 2024-11-17T03:15:41,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T03:15:41,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731813341454, max timestamp=1731813341462Num uniques 1104; 2024-11-17T03:15:41,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-17T03:15:41,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731813341454, max timestamp=1731813341462Num uniques 1104; 2024-11-17T03:15:41,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-17T03:15:41,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731813341454, max timestamp=1731813341462Num uniques 1104; 2024-11-17T03:15:41,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-17T03:15:41,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731813341477, max timestamp=1731813341483Num uniques 1113; 2024-11-17T03:15:41,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-17T03:15:41,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731813341477, max timestamp=1731813341483Num uniques 1113; 2024-11-17T03:15:41,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-17T03:15:41,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1731813341477, max timestamp=1731813341483Num uniques 1113; 2024-11-17T03:15:41,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=12038, suffix heap size=143720,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=263 2024-11-17T03:15:41,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813401503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813401504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731813341491, max timestamp=1731813341508Num uniques 1122; 2024-11-17T03:15:41,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-11-17T03:15:41,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10159, totalHeapSize=23328, min timestamp=1731813341491, max timestamp=1731813341509Num uniques 1122; 2024-11-17T03:15:41,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51379, new segment data size=10159, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-11-17T03:15:41,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10152, totalHeapSize=23328, min timestamp=1731813341491, max timestamp=1731813341509Num uniques 1122; 2024-11-17T03:15:41,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51372, new segment data size=10152, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-11-17T03:15:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731813341522, max timestamp=1731813341542Num uniques 1081; 2024-11-17T03:15:41,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-17T03:15:41,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731813341522, max timestamp=1731813341542Num uniques 1081; 2024-11-17T03:15:41,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-17T03:15:41,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731813341522, max timestamp=1731813341542Num uniques 1081; 2024-11-17T03:15:41,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-17T03:15:41,551 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/355f3254157742c6a79fe21a2cd8beb1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/355f3254157742c6a79fe21a2cd8beb1 2024-11-17T03:15:41,556 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into 355f3254157742c6a79fe21a2cd8beb1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:41,556 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:41,556 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=13, startTime=1731813341092; duration=0sec 2024-11-17T03:15:41,556 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:41,556 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T03:15:41,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731813341563, max timestamp=1731813341584Num uniques 1090; 2024-11-17T03:15:41,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731813341563, max timestamp=1731813341584Num uniques 1090; 2024-11-17T03:15:41,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-17T03:15:41,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-17T03:15:41,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731813341563, max timestamp=1731813341584Num uniques 1090; 2024-11-17T03:15:41,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=8700, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-17T03:15:41,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731813341625, max timestamp=1731813341634Num uniques 1049; 2024-11-17T03:15:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-17T03:15:41,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731813341625, max timestamp=1731813341634Num uniques 1049; 2024-11-17T03:15:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-17T03:15:41,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731813341625, max timestamp=1731813341634Num uniques 1049; 2024-11-17T03:15:41,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-17T03:15:41,640 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/cc7454b21db249f7ad405135f59c2db0 2024-11-17T03:15:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/8b4fe41ec4764f9094c4ce79a4795f50 is 50, key is test_row_0/B:col10/1731813340208/Put/seqid=0 2024-11-17T03:15:41,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813401668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731813341664, max timestamp=1731813341672Num uniques 1108; 2024-11-17T03:15:41,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-17T03:15:41,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731813341664, max timestamp=1731813341672Num uniques 1108; 2024-11-17T03:15:41,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-17T03:15:41,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731813341665, max timestamp=1731813341672Num uniques 1108; 2024-11-17T03:15:41,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=6870, suffix heap size=143200,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=150 2024-11-17T03:15:41,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742066_1242 (size=9757) 2024-11-17T03:15:41,695 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/8b4fe41ec4764f9094c4ce79a4795f50 2024-11-17T03:15:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4f82fa80f65f485eb83f3ab43ed151f1 is 50, key is test_row_0/C:col10/1731813340208/Put/seqid=0 2024-11-17T03:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813341711, max timestamp=1731813341725Num uniques 1067; 2024-11-17T03:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-17T03:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813341711, max timestamp=1731813341725Num uniques 1067; 2024-11-17T03:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-17T03:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813341711, max timestamp=1731813341725Num uniques 1009; 2024-11-17T03:15:41,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=9932, suffix heap size=132904,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=217 2024-11-17T03:15:41,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742067_1243 (size=9757) 2024-11-17T03:15:41,739 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4f82fa80f65f485eb83f3ab43ed151f1 2024-11-17T03:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/cc7454b21db249f7ad405135f59c2db0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/cc7454b21db249f7ad405135f59c2db0 2024-11-17T03:15:41,751 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/cc7454b21db249f7ad405135f59c2db0, entries=100, sequenceid=153, filesize=9.5 K 2024-11-17T03:15:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/8b4fe41ec4764f9094c4ce79a4795f50 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8b4fe41ec4764f9094c4ce79a4795f50 2024-11-17T03:15:41,758 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8b4fe41ec4764f9094c4ce79a4795f50, entries=100, sequenceid=153, filesize=9.5 K 2024-11-17T03:15:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4f82fa80f65f485eb83f3ab43ed151f1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4f82fa80f65f485eb83f3ab43ed151f1 2024-11-17T03:15:41,765 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4f82fa80f65f485eb83f3ab43ed151f1, entries=100, sequenceid=153, filesize=9.5 K 2024-11-17T03:15:41,788 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=60.38 KB/61830 for a5d5a03df69af3fa2659ea9597853e20 in 585ms, sequenceid=153, compaction requested=false 2024-11-17T03:15:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T03:15:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:41,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=80.51 KB heapSize=220.17 KB 2024-11-17T03:15:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T03:15:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:41,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:41,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T03:15:41,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3500 sec 2024-11-17T03:15:41,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.3570 sec 2024-11-17T03:15:41,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/3a4c6f811bd94e52a1f91bc9004ae6ee is 50, key is test_row_0/A:col10/1731813341736/Put/seqid=0 2024-11-17T03:15:41,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742068_1244 (size=12301) 2024-11-17T03:15:41,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813401842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813401841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813401842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813401844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813401957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813401957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813401957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813401958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813401958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813402187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813402187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813402187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813402187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813402188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/3a4c6f811bd94e52a1f91bc9004ae6ee 2024-11-17T03:15:42,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/ee6dc6498a8c428fa91ef1ad56781cc5 is 50, key is test_row_0/B:col10/1731813341736/Put/seqid=0 2024-11-17T03:15:42,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742069_1245 (size=12301) 2024-11-17T03:15:42,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813402497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813402497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813402497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813402497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813402497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T03:15:42,587 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:42,588 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T03:15:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T03:15:42,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:42,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:42,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:42,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/ee6dc6498a8c428fa91ef1ad56781cc5 2024-11-17T03:15:42,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/9ba9cf5ec0b54859aa04b09cf89b62b8 is 50, key is test_row_0/C:col10/1731813341736/Put/seqid=0 2024-11-17T03:15:42,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742070_1246 (size=12301) 2024-11-17T03:15:42,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/9ba9cf5ec0b54859aa04b09cf89b62b8 2024-11-17T03:15:42,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/3a4c6f811bd94e52a1f91bc9004ae6ee as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/3a4c6f811bd94e52a1f91bc9004ae6ee 2024-11-17T03:15:42,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/3a4c6f811bd94e52a1f91bc9004ae6ee, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T03:15:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/ee6dc6498a8c428fa91ef1ad56781cc5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/ee6dc6498a8c428fa91ef1ad56781cc5 2024-11-17T03:15:42,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/ee6dc6498a8c428fa91ef1ad56781cc5, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T03:15:42,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/9ba9cf5ec0b54859aa04b09cf89b62b8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/9ba9cf5ec0b54859aa04b09cf89b62b8 2024-11-17T03:15:42,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/9ba9cf5ec0b54859aa04b09cf89b62b8, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T03:15:42,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~220.13 KB/225408, currentSize=100.63 KB/103050 for a5d5a03df69af3fa2659ea9597853e20 in 884ms, sequenceid=388, compaction requested=true 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:42,673 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:42,673 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:42,674 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:42,674 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:42,674 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:42,674 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:42,674 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:42,674 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:42,674 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/355f3254157742c6a79fe21a2cd8beb1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/cc7454b21db249f7ad405135f59c2db0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/3a4c6f811bd94e52a1f91bc9004ae6ee] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=33.6 K 2024-11-17T03:15:42,674 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4190b1df2e7c4b27886d155134c8ed1b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8b4fe41ec4764f9094c4ce79a4795f50, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/ee6dc6498a8c428fa91ef1ad56781cc5] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=33.6 K 2024-11-17T03:15:42,675 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4190b1df2e7c4b27886d155134c8ed1b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731813339053 2024-11-17T03:15:42,675 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355f3254157742c6a79fe21a2cd8beb1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731813339053 2024-11-17T03:15:42,675 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4fe41ec4764f9094c4ce79a4795f50, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731813340205 2024-11-17T03:15:42,675 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7454b21db249f7ad405135f59c2db0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731813340205 2024-11-17T03:15:42,676 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a4c6f811bd94e52a1f91bc9004ae6ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731813341730 2024-11-17T03:15:42,676 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6dc6498a8c428fa91ef1ad56781cc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731813341730 2024-11-17T03:15:42,685 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:42,685 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/11c461a30a79416e86a3f09a6e61b1a1 is 50, key is test_row_0/A:col10/1731813341736/Put/seqid=0 2024-11-17T03:15:42,687 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:42,688 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/8460fe1762ed4637b2f3338b8ecc3a0e is 50, key is test_row_0/B:col10/1731813341736/Put/seqid=0 2024-11-17T03:15:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T03:15:42,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742072_1248 (size=12643) 2024-11-17T03:15:42,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742071_1247 (size=12643) 2024-11-17T03:15:42,718 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/8460fe1762ed4637b2f3338b8ecc3a0e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8460fe1762ed4637b2f3338b8ecc3a0e 2024-11-17T03:15:42,731 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into 8460fe1762ed4637b2f3338b8ecc3a0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:42,731 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:42,731 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=13, startTime=1731813342673; duration=0sec 2024-11-17T03:15:42,731 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:42,731 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:42,731 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:42,734 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:42,734 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:42,734 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:42,734 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/07dfdf0ef439434eadb164d8be066e17, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4f82fa80f65f485eb83f3ab43ed151f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/9ba9cf5ec0b54859aa04b09cf89b62b8] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=33.6 K 2024-11-17T03:15:42,734 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 07dfdf0ef439434eadb164d8be066e17, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731813339053 2024-11-17T03:15:42,735 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f82fa80f65f485eb83f3ab43ed151f1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731813340205 2024-11-17T03:15:42,735 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ba9cf5ec0b54859aa04b09cf89b62b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731813341730 2024-11-17T03:15:42,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T03:15:42,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:42,743 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:15:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:42,748 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:42,749 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4040a43112174094a839680c6e1f5aad is 50, key is test_row_0/C:col10/1731813341736/Put/seqid=0 2024-11-17T03:15:42,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/d4d3532e94ac4a658c040d055675ee15 is 50, key is test_row_0/A:col10/1731813341838/Put/seqid=0 2024-11-17T03:15:42,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742073_1249 (size=12643) 2024-11-17T03:15:42,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742074_1250 (size=12301) 2024-11-17T03:15:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T03:15:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:43,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813403022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813403032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813403054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813403054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813403054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,114 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/11c461a30a79416e86a3f09a6e61b1a1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/11c461a30a79416e86a3f09a6e61b1a1 2024-11-17T03:15:43,118 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into 11c461a30a79416e86a3f09a6e61b1a1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:43,119 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:43,119 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=13, startTime=1731813342673; duration=0sec 2024-11-17T03:15:43,119 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:43,119 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813403167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813403167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813403167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813403167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813403168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,183 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4040a43112174094a839680c6e1f5aad as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4040a43112174094a839680c6e1f5aad 2024-11-17T03:15:43,187 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into 4040a43112174094a839680c6e1f5aad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:43,187 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:43,187 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=13, startTime=1731813342673; duration=0sec 2024-11-17T03:15:43,187 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:43,187 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:43,190 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/d4d3532e94ac4a658c040d055675ee15 2024-11-17T03:15:43,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4d1c68e712644946931a4b46d716c6a1 is 50, key is test_row_0/B:col10/1731813341838/Put/seqid=0 2024-11-17T03:15:43,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742075_1251 (size=12301) 2024-11-17T03:15:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T03:15:43,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813403377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813403378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813403378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813403378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813403378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,602 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4d1c68e712644946931a4b46d716c6a1 2024-11-17T03:15:43,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/2b6e527627c7427d85cc74c832f49ba8 is 50, key is test_row_0/C:col10/1731813341838/Put/seqid=0 2024-11-17T03:15:43,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742076_1252 (size=12301) 2024-11-17T03:15:43,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813403687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813403687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813403687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813403687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813403687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T03:15:44,013 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/2b6e527627c7427d85cc74c832f49ba8 2024-11-17T03:15:44,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/d4d3532e94ac4a658c040d055675ee15 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d4d3532e94ac4a658c040d055675ee15 2024-11-17T03:15:44,021 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d4d3532e94ac4a658c040d055675ee15, entries=150, sequenceid=407, filesize=12.0 K 2024-11-17T03:15:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/4d1c68e712644946931a4b46d716c6a1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4d1c68e712644946931a4b46d716c6a1 2024-11-17T03:15:44,026 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4d1c68e712644946931a4b46d716c6a1, entries=150, sequenceid=407, filesize=12.0 K 2024-11-17T03:15:44,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/2b6e527627c7427d85cc74c832f49ba8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2b6e527627c7427d85cc74c832f49ba8 2024-11-17T03:15:44,031 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2b6e527627c7427d85cc74c832f49ba8, entries=150, sequenceid=407, filesize=12.0 K 2024-11-17T03:15:44,032 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a5d5a03df69af3fa2659ea9597853e20 in 1290ms, sequenceid=407, compaction requested=false 2024-11-17T03:15:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:44,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T03:15:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T03:15:44,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T03:15:44,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4430 sec 2024-11-17T03:15:44,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4470 sec 2024-11-17T03:15:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:44,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:15:44,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:44,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:44,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:44,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:44,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:44,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:44,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/89bd9f8d7809428baf20873c806228e2 is 50, key is test_row_0/A:col10/1731813344197/Put/seqid=0 2024-11-17T03:15:44,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742077_1253 (size=14741) 2024-11-17T03:15:44,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/89bd9f8d7809428baf20873c806228e2 2024-11-17T03:15:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813404213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813404216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813404219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813404219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813404220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/eb637c0ff0904d8b9251d932b145936c is 50, key is test_row_0/B:col10/1731813344197/Put/seqid=0 2024-11-17T03:15:44,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742078_1254 (size=12301) 2024-11-17T03:15:44,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/eb637c0ff0904d8b9251d932b145936c 2024-11-17T03:15:44,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/069d6a8887e84be09d982eacdf028930 is 50, key is test_row_0/C:col10/1731813344197/Put/seqid=0 2024-11-17T03:15:44,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742079_1255 (size=12301) 2024-11-17T03:15:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813404327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813404327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813404327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813404327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813404337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813404537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813404537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813404537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813404537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813404547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/069d6a8887e84be09d982eacdf028930 2024-11-17T03:15:44,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/89bd9f8d7809428baf20873c806228e2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/89bd9f8d7809428baf20873c806228e2 2024-11-17T03:15:44,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/89bd9f8d7809428baf20873c806228e2, entries=200, sequenceid=424, filesize=14.4 K 2024-11-17T03:15:44,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/eb637c0ff0904d8b9251d932b145936c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/eb637c0ff0904d8b9251d932b145936c 2024-11-17T03:15:44,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/eb637c0ff0904d8b9251d932b145936c, entries=150, sequenceid=424, filesize=12.0 K 2024-11-17T03:15:44,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/069d6a8887e84be09d982eacdf028930 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/069d6a8887e84be09d982eacdf028930 2024-11-17T03:15:44,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/069d6a8887e84be09d982eacdf028930, entries=150, sequenceid=424, filesize=12.0 K 2024-11-17T03:15:44,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a5d5a03df69af3fa2659ea9597853e20 in 466ms, sequenceid=424, compaction requested=true 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:44,665 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:44,665 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:44,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:44,666 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:44,666 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:44,666 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:44,666 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:44,666 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:44,666 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:44,666 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/11c461a30a79416e86a3f09a6e61b1a1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d4d3532e94ac4a658c040d055675ee15, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/89bd9f8d7809428baf20873c806228e2] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=38.8 K 2024-11-17T03:15:44,666 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8460fe1762ed4637b2f3338b8ecc3a0e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4d1c68e712644946931a4b46d716c6a1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/eb637c0ff0904d8b9251d932b145936c] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=36.4 K 2024-11-17T03:15:44,666 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8460fe1762ed4637b2f3338b8ecc3a0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731813341730 2024-11-17T03:15:44,666 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c461a30a79416e86a3f09a6e61b1a1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731813341730 2024-11-17T03:15:44,667 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1c68e712644946931a4b46d716c6a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731813341827 2024-11-17T03:15:44,667 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4d3532e94ac4a658c040d055675ee15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731813341827 2024-11-17T03:15:44,667 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89bd9f8d7809428baf20873c806228e2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731813343021 2024-11-17T03:15:44,667 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting eb637c0ff0904d8b9251d932b145936c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731813343021 2024-11-17T03:15:44,674 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:44,674 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:44,674 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/c919904f09b144b4be3c0d8048f341df is 50, key is test_row_0/A:col10/1731813344197/Put/seqid=0 2024-11-17T03:15:44,674 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/afbdc5b9d9184c5a8f7c7c140eb849d5 is 50, key is test_row_0/B:col10/1731813344197/Put/seqid=0 2024-11-17T03:15:44,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742080_1256 (size=12745) 2024-11-17T03:15:44,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742081_1257 (size=12745) 2024-11-17T03:15:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T03:15:44,737 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:44,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T03:15:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:44,739 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:44,740 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:44,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:44,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:15:44,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:44,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:44,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:44,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:44,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:44,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:44,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/796fc5f78c414c20934a7a26df0c92fd is 50, key is test_row_0/A:col10/1731813344216/Put/seqid=0 2024-11-17T03:15:44,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742082_1258 (size=12301) 2024-11-17T03:15:44,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813404861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813404862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813404862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813404865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813404867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:44,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:44,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:44,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813404977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813404977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813404977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813404977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813404977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:45,091 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/c919904f09b144b4be3c0d8048f341df as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/c919904f09b144b4be3c0d8048f341df 2024-11-17T03:15:45,095 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into c919904f09b144b4be3c0d8048f341df(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:45,095 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:45,095 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=13, startTime=1731813344665; duration=0sec 2024-11-17T03:15:45,096 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:45,096 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:45,096 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:45,097 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:45,097 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:45,097 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,097 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4040a43112174094a839680c6e1f5aad, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2b6e527627c7427d85cc74c832f49ba8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/069d6a8887e84be09d982eacdf028930] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=36.4 K 2024-11-17T03:15:45,098 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4040a43112174094a839680c6e1f5aad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731813341730 2024-11-17T03:15:45,100 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b6e527627c7427d85cc74c832f49ba8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731813341827 2024-11-17T03:15:45,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 069d6a8887e84be09d982eacdf028930, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731813343021 2024-11-17T03:15:45,105 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/afbdc5b9d9184c5a8f7c7c140eb849d5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/afbdc5b9d9184c5a8f7c7c140eb849d5 2024-11-17T03:15:45,112 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:45,113 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into afbdc5b9d9184c5a8f7c7c140eb849d5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:45,113 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:45,113 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=13, startTime=1731813344665; duration=0sec 2024-11-17T03:15:45,113 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:45,113 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:45,113 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/7555c0393c1646b692e2cc0de41d5d14 is 50, key is test_row_0/C:col10/1731813344197/Put/seqid=0 2024-11-17T03:15:45,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742083_1259 (size=12745) 2024-11-17T03:15:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813405187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813405187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813405187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813405187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813405187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/796fc5f78c414c20934a7a26df0c92fd 2024-11-17T03:15:45,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/26a3cc992ec94cbea81507439449f6a6 is 50, key is test_row_0/B:col10/1731813344216/Put/seqid=0 2024-11-17T03:15:45,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742084_1260 (size=12301) 2024-11-17T03:15:45,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:45,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813405497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813405497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813405497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813405497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813405497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:45,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,530 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/7555c0393c1646b692e2cc0de41d5d14 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/7555c0393c1646b692e2cc0de41d5d14 2024-11-17T03:15:45,535 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into 7555c0393c1646b692e2cc0de41d5d14(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:45,535 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:45,535 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=13, startTime=1731813344665; duration=0sec 2024-11-17T03:15:45,535 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:45,535 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:45,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/26a3cc992ec94cbea81507439449f6a6 2024-11-17T03:15:45,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/8379b3a411814e939a4a7ed29187da25 is 50, key is test_row_0/C:col10/1731813344216/Put/seqid=0 2024-11-17T03:15:45,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742085_1261 (size=12301) 2024-11-17T03:15:45,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:45,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:45,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:46,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813406007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:46,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813406007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813406007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813406008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:46,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813406008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:46,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/8379b3a411814e939a4a7ed29187da25 2024-11-17T03:15:46,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/796fc5f78c414c20934a7a26df0c92fd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/796fc5f78c414c20934a7a26df0c92fd 2024-11-17T03:15:46,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/796fc5f78c414c20934a7a26df0c92fd, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T03:15:46,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/26a3cc992ec94cbea81507439449f6a6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26a3cc992ec94cbea81507439449f6a6 2024-11-17T03:15:46,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:46,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:46,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26a3cc992ec94cbea81507439449f6a6, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T03:15:46,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/8379b3a411814e939a4a7ed29187da25 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8379b3a411814e939a4a7ed29187da25 2024-11-17T03:15:46,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8379b3a411814e939a4a7ed29187da25, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T03:15:46,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a5d5a03df69af3fa2659ea9597853e20 in 1277ms, sequenceid=442, compaction requested=false 2024-11-17T03:15:46,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:46,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:46,270 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:46,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/8f538ad6cbc54ab0bc574ba101521ba5 is 50, key is test_row_0/A:col10/1731813344865/Put/seqid=0 2024-11-17T03:15:46,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742086_1262 (size=12301) 2024-11-17T03:15:46,282 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/8f538ad6cbc54ab0bc574ba101521ba5 2024-11-17T03:15:46,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/0a32b275bc764569a82bb896eabcc0e6 is 50, key is test_row_0/B:col10/1731813344865/Put/seqid=0 2024-11-17T03:15:46,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742087_1263 (size=12301) 2024-11-17T03:15:46,693 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/0a32b275bc764569a82bb896eabcc0e6 2024-11-17T03:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/bc3a33d331e049c3b08fa3ae6b15b4cf is 50, key is test_row_0/C:col10/1731813344865/Put/seqid=0 2024-11-17T03:15:46,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742088_1264 (size=12301) 2024-11-17T03:15:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813407039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813407048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813407080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813407080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813407080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,104 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/bc3a33d331e049c3b08fa3ae6b15b4cf 2024-11-17T03:15:47,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/8f538ad6cbc54ab0bc574ba101521ba5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/8f538ad6cbc54ab0bc574ba101521ba5 2024-11-17T03:15:47,112 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/8f538ad6cbc54ab0bc574ba101521ba5, entries=150, sequenceid=459, filesize=12.0 K 2024-11-17T03:15:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/0a32b275bc764569a82bb896eabcc0e6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/0a32b275bc764569a82bb896eabcc0e6 2024-11-17T03:15:47,117 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/0a32b275bc764569a82bb896eabcc0e6, entries=150, sequenceid=459, filesize=12.0 K 2024-11-17T03:15:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/bc3a33d331e049c3b08fa3ae6b15b4cf as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/bc3a33d331e049c3b08fa3ae6b15b4cf 2024-11-17T03:15:47,123 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/bc3a33d331e049c3b08fa3ae6b15b4cf, entries=150, sequenceid=459, filesize=12.0 K 2024-11-17T03:15:47,124 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a5d5a03df69af3fa2659ea9597853e20 in 854ms, sequenceid=459, compaction requested=true 2024-11-17T03:15:47,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:47,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:47,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T03:15:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T03:15:47,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T03:15:47,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3860 sec 2024-11-17T03:15:47,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.3910 sec 2024-11-17T03:15:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:47,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T03:15:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:47,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:47,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/48cd7f6b4d784a9997594f3d84d75fc2 is 50, key is test_row_0/A:col10/1731813347045/Put/seqid=0 2024-11-17T03:15:47,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813407203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742089_1265 (size=14741) 2024-11-17T03:15:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813407204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813407205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813407207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813407208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813407317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813407317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813407317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813407317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813407317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813407527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813407527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813407527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813407527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813407527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/48cd7f6b4d784a9997594f3d84d75fc2 2024-11-17T03:15:47,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/c08ac03526b943bf901484ddee3d1214 is 50, key is test_row_0/B:col10/1731813347045/Put/seqid=0 2024-11-17T03:15:47,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742090_1266 (size=12301) 2024-11-17T03:15:47,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813407837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813407837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813407837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813407838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813407838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/c08ac03526b943bf901484ddee3d1214 2024-11-17T03:15:48,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/83a5f2f7d1e84598b5ed548fd61cef8d is 50, key is test_row_0/C:col10/1731813347045/Put/seqid=0 2024-11-17T03:15:48,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742091_1267 (size=12301) 2024-11-17T03:15:48,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/83a5f2f7d1e84598b5ed548fd61cef8d 2024-11-17T03:15:48,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/48cd7f6b4d784a9997594f3d84d75fc2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/48cd7f6b4d784a9997594f3d84d75fc2 2024-11-17T03:15:48,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/48cd7f6b4d784a9997594f3d84d75fc2, entries=200, sequenceid=479, filesize=14.4 K 2024-11-17T03:15:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/c08ac03526b943bf901484ddee3d1214 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c08ac03526b943bf901484ddee3d1214 2024-11-17T03:15:48,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c08ac03526b943bf901484ddee3d1214, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T03:15:48,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/83a5f2f7d1e84598b5ed548fd61cef8d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/83a5f2f7d1e84598b5ed548fd61cef8d 2024-11-17T03:15:48,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/83a5f2f7d1e84598b5ed548fd61cef8d, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T03:15:48,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a5d5a03df69af3fa2659ea9597853e20 in 905ms, sequenceid=479, compaction requested=true 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:48,095 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:48,095 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:48,097 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:48,097 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:48,097 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:48,097 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/afbdc5b9d9184c5a8f7c7c140eb849d5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26a3cc992ec94cbea81507439449f6a6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/0a32b275bc764569a82bb896eabcc0e6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c08ac03526b943bf901484ddee3d1214] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=48.5 K 2024-11-17T03:15:48,100 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting afbdc5b9d9184c5a8f7c7c140eb849d5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731813343021 2024-11-17T03:15:48,100 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:48,100 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:48,100 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:48,100 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/c919904f09b144b4be3c0d8048f341df, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/796fc5f78c414c20934a7a26df0c92fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/8f538ad6cbc54ab0bc574ba101521ba5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/48cd7f6b4d784a9997594f3d84d75fc2] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=50.9 K 2024-11-17T03:15:48,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a3cc992ec94cbea81507439449f6a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731813344214 2024-11-17T03:15:48,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c919904f09b144b4be3c0d8048f341df, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731813343021 2024-11-17T03:15:48,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a32b275bc764569a82bb896eabcc0e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731813344861 2024-11-17T03:15:48,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796fc5f78c414c20934a7a26df0c92fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731813344214 2024-11-17T03:15:48,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c08ac03526b943bf901484ddee3d1214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731813347033 2024-11-17T03:15:48,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f538ad6cbc54ab0bc574ba101521ba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731813344861 2024-11-17T03:15:48,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48cd7f6b4d784a9997594f3d84d75fc2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731813347033 2024-11-17T03:15:48,120 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:48,120 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/c93b66dc706848ef986ecfc622944cee is 50, key is test_row_0/B:col10/1731813347045/Put/seqid=0 2024-11-17T03:15:48,127 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:48,128 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/d9e5097bc09940a896fab75ded634ff7 is 50, key is test_row_0/A:col10/1731813347045/Put/seqid=0 2024-11-17T03:15:48,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742092_1268 (size=12881) 2024-11-17T03:15:48,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742093_1269 (size=12881) 2024-11-17T03:15:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:48,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:48,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/955aee62d8e746d08d445e50a904fdfe is 50, key is test_row_0/A:col10/1731813348349/Put/seqid=0 2024-11-17T03:15:48,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742094_1270 (size=19621) 2024-11-17T03:15:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813408370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813408371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813408371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813408372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813408378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813408486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813408486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813408487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813408487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813408487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,538 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/c93b66dc706848ef986ecfc622944cee as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c93b66dc706848ef986ecfc622944cee 2024-11-17T03:15:48,544 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into c93b66dc706848ef986ecfc622944cee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:48,544 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:48,544 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=12, startTime=1731813348095; duration=0sec 2024-11-17T03:15:48,544 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:48,544 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:48,545 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:48,546 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:48,546 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:48,546 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:48,546 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/7555c0393c1646b692e2cc0de41d5d14, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8379b3a411814e939a4a7ed29187da25, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/bc3a33d331e049c3b08fa3ae6b15b4cf, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/83a5f2f7d1e84598b5ed548fd61cef8d] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=48.5 K 2024-11-17T03:15:48,547 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7555c0393c1646b692e2cc0de41d5d14, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731813343021 2024-11-17T03:15:48,547 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8379b3a411814e939a4a7ed29187da25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731813344214 2024-11-17T03:15:48,547 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting bc3a33d331e049c3b08fa3ae6b15b4cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731813344861 2024-11-17T03:15:48,547 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/d9e5097bc09940a896fab75ded634ff7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d9e5097bc09940a896fab75ded634ff7 2024-11-17T03:15:48,547 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a5f2f7d1e84598b5ed548fd61cef8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731813347033 2024-11-17T03:15:48,551 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into d9e5097bc09940a896fab75ded634ff7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:48,551 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:48,552 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=12, startTime=1731813348095; duration=0sec 2024-11-17T03:15:48,552 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:48,552 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:48,555 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:48,556 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/18abfacf7f8f4d7d8760cdf645b67f05 is 50, key is test_row_0/C:col10/1731813347045/Put/seqid=0 2024-11-17T03:15:48,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742095_1271 (size=12881) 2024-11-17T03:15:48,574 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/18abfacf7f8f4d7d8760cdf645b67f05 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/18abfacf7f8f4d7d8760cdf645b67f05 2024-11-17T03:15:48,580 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into 18abfacf7f8f4d7d8760cdf645b67f05(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:48,580 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:48,580 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=12, startTime=1731813348095; duration=0sec 2024-11-17T03:15:48,580 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:48,580 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:48,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813408696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813408697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813408697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813408697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813408697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:48,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/955aee62d8e746d08d445e50a904fdfe 2024-11-17T03:15:48,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/5081ed4a75404e66b1688235022eca78 is 50, key is test_row_0/B:col10/1731813348349/Put/seqid=0 2024-11-17T03:15:48,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742096_1272 (size=12301) 2024-11-17T03:15:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T03:15:48,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:48,897 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T03:15:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T03:15:48,899 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:48,899 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:48,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T03:15:49,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813409007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813409008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T03:15:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/5081ed4a75404e66b1688235022eca78 2024-11-17T03:15:49,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/547121878b834ef088e0099d4f3aa285 is 50, key is test_row_0/C:col10/1731813348349/Put/seqid=0 2024-11-17T03:15:49,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742097_1273 (size=12301) 2024-11-17T03:15:49,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T03:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T03:15:49,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T03:15:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T03:15:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813409517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813409517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813409517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T03:15:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813409527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813409527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:49,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/547121878b834ef088e0099d4f3aa285 2024-11-17T03:15:49,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/955aee62d8e746d08d445e50a904fdfe as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/955aee62d8e746d08d445e50a904fdfe 2024-11-17T03:15:49,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/955aee62d8e746d08d445e50a904fdfe, entries=300, sequenceid=494, filesize=19.2 K 2024-11-17T03:15:49,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/5081ed4a75404e66b1688235022eca78 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/5081ed4a75404e66b1688235022eca78 2024-11-17T03:15:49,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/5081ed4a75404e66b1688235022eca78, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T03:15:49,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/547121878b834ef088e0099d4f3aa285 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/547121878b834ef088e0099d4f3aa285 2024-11-17T03:15:49,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/547121878b834ef088e0099d4f3aa285, entries=150, sequenceid=494, filesize=12.0 K 2024-11-17T03:15:49,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a5d5a03df69af3fa2659ea9597853e20 in 1255ms, sequenceid=494, compaction requested=false 2024-11-17T03:15:49,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:49,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T03:15:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:49,661 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:15:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:49,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:49,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/fbe1850d75644e55b268d6b3ecaf90da is 50, key is test_row_0/A:col10/1731813348376/Put/seqid=0 2024-11-17T03:15:49,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742098_1274 (size=12301) 2024-11-17T03:15:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T03:15:50,072 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/fbe1850d75644e55b268d6b3ecaf90da 2024-11-17T03:15:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/d31f0097f98c47919369e4a41c455d2d is 50, key is test_row_0/B:col10/1731813348376/Put/seqid=0 2024-11-17T03:15:50,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742099_1275 (size=12301) 2024-11-17T03:15:50,485 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/d31f0097f98c47919369e4a41c455d2d 2024-11-17T03:15:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/8cb9242d2a43446baa8edcd745b4bfae is 50, key is test_row_0/C:col10/1731813348376/Put/seqid=0 2024-11-17T03:15:50,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742100_1276 (size=12301) 2024-11-17T03:15:50,496 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/8cb9242d2a43446baa8edcd745b4bfae 2024-11-17T03:15:50,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/fbe1850d75644e55b268d6b3ecaf90da as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/fbe1850d75644e55b268d6b3ecaf90da 2024-11-17T03:15:50,507 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/fbe1850d75644e55b268d6b3ecaf90da, entries=150, sequenceid=514, filesize=12.0 K 2024-11-17T03:15:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-17T03:15:50,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/d31f0097f98c47919369e4a41c455d2d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/d31f0097f98c47919369e4a41c455d2d 2024-11-17T03:15:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/d31f0097f98c47919369e4a41c455d2d, entries=150, sequenceid=514, filesize=12.0 K 2024-11-17T03:15:50,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/8cb9242d2a43446baa8edcd745b4bfae as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8cb9242d2a43446baa8edcd745b4bfae 2024-11-17T03:15:50,518 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8cb9242d2a43446baa8edcd745b4bfae, entries=150, sequenceid=514, filesize=12.0 K 2024-11-17T03:15:50,519 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for a5d5a03df69af3fa2659ea9597853e20 in 858ms, sequenceid=514, compaction requested=true 2024-11-17T03:15:50,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:50,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:50,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T03:15:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T03:15:50,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T03:15:50,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-11-17T03:15:50,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6260 sec 2024-11-17T03:15:50,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:50,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:15:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:50,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/0048f5707a864e9e8a50f915e0f1005e is 50, key is test_row_0/A:col10/1731813350534/Put/seqid=0 2024-11-17T03:15:50,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742101_1277 (size=14741) 2024-11-17T03:15:50,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813350553, max timestamp=1731813350568Num uniques 859; 2024-11-17T03:15:50,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:15:50,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813350553, max timestamp=1731813350568Num uniques 859; 2024-11-17T03:15:50,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:15:50,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813350553, max timestamp=1731813350568Num uniques 859; 2024-11-17T03:15:50,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:15:50,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813410594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813410597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813410598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813410602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813410603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813410717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813410717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813410717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813410717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813410718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813410927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813410927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813410927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813410927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813410927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:50,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:50,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/0048f5707a864e9e8a50f915e0f1005e 2024-11-17T03:15:50,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/96cce80083804a0c8276f861c05f62bc is 50, key is test_row_0/B:col10/1731813350534/Put/seqid=0 2024-11-17T03:15:51,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742102_1278 (size=12301) 2024-11-17T03:15:51,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/96cce80083804a0c8276f861c05f62bc 2024-11-17T03:15:51,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/412bafbbae3e4598983cdf56afb171c6 is 50, key is test_row_0/C:col10/1731813350534/Put/seqid=0 2024-11-17T03:15:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T03:15:51,048 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:51,050 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T03:15:51,053 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:51,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742103_1279 (size=12301) 2024-11-17T03:15:51,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T03:15:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T03:15:51,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T03:15:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:51,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813411237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813411237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813411237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813411237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813411247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T03:15:51,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:51,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:51,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T03:15:51,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/412bafbbae3e4598983cdf56afb171c6 2024-11-17T03:15:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/0048f5707a864e9e8a50f915e0f1005e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0048f5707a864e9e8a50f915e0f1005e 2024-11-17T03:15:51,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0048f5707a864e9e8a50f915e0f1005e, entries=200, sequenceid=525, filesize=14.4 K 2024-11-17T03:15:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/96cce80083804a0c8276f861c05f62bc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/96cce80083804a0c8276f861c05f62bc 2024-11-17T03:15:51,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/96cce80083804a0c8276f861c05f62bc, entries=150, sequenceid=525, filesize=12.0 K 2024-11-17T03:15:51,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/412bafbbae3e4598983cdf56afb171c6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/412bafbbae3e4598983cdf56afb171c6 2024-11-17T03:15:51,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/412bafbbae3e4598983cdf56afb171c6, entries=150, sequenceid=525, filesize=12.0 K 2024-11-17T03:15:51,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a5d5a03df69af3fa2659ea9597853e20 in 968ms, sequenceid=525, compaction requested=true 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:51,503 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:51,503 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:51,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:51,508 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:51,508 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:51,508 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,508 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c93b66dc706848ef986ecfc622944cee, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/5081ed4a75404e66b1688235022eca78, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/d31f0097f98c47919369e4a41c455d2d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/96cce80083804a0c8276f861c05f62bc] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=48.6 K 2024-11-17T03:15:51,509 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:51,509 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:51,509 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,509 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d9e5097bc09940a896fab75ded634ff7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/955aee62d8e746d08d445e50a904fdfe, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/fbe1850d75644e55b268d6b3ecaf90da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0048f5707a864e9e8a50f915e0f1005e] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=58.1 K 2024-11-17T03:15:51,510 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c93b66dc706848ef986ecfc622944cee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731813347033 2024-11-17T03:15:51,510 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e5097bc09940a896fab75ded634ff7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731813347033 2024-11-17T03:15:51,511 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5081ed4a75404e66b1688235022eca78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731813348349 2024-11-17T03:15:51,511 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 955aee62d8e746d08d445e50a904fdfe, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731813347201 2024-11-17T03:15:51,511 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting d31f0097f98c47919369e4a41c455d2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731813348370 2024-11-17T03:15:51,511 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe1850d75644e55b268d6b3ecaf90da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731813348370 2024-11-17T03:15:51,512 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 96cce80083804a0c8276f861c05f62bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731813350528 2024-11-17T03:15:51,512 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0048f5707a864e9e8a50f915e0f1005e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731813350528 2024-11-17T03:15:51,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T03:15:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,521 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-17T03:15:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:15:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/e34b34baa67b4706b1e51b42c085e829 is 50, key is test_row_0/A:col10/1731813350578/Put/seqid=0 2024-11-17T03:15:51,556 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:51,556 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/26602e8cb6db45819fdb3259a5208f14 is 50, key is test_row_0/B:col10/1731813350534/Put/seqid=0 2024-11-17T03:15:51,561 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#242 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:51,561 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/7ebd29ca0250470f98ec941ec04d6aed is 50, key is test_row_0/A:col10/1731813350534/Put/seqid=0 2024-11-17T03:15:51,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742104_1280 (size=12301) 2024-11-17T03:15:51,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742106_1282 (size=13017) 2024-11-17T03:15:51,645 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/7ebd29ca0250470f98ec941ec04d6aed as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/7ebd29ca0250470f98ec941ec04d6aed 2024-11-17T03:15:51,652 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into 7ebd29ca0250470f98ec941ec04d6aed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:51,652 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:51,652 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=12, startTime=1731813351503; duration=0sec 2024-11-17T03:15:51,653 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:51,653 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:51,653 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:15:51,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742105_1281 (size=13017) 2024-11-17T03:15:51,656 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:15:51,656 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:51,656 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:51,657 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/18abfacf7f8f4d7d8760cdf645b67f05, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/547121878b834ef088e0099d4f3aa285, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8cb9242d2a43446baa8edcd745b4bfae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/412bafbbae3e4598983cdf56afb171c6] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=48.6 K 2024-11-17T03:15:51,657 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18abfacf7f8f4d7d8760cdf645b67f05, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731813347033 2024-11-17T03:15:51,657 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 547121878b834ef088e0099d4f3aa285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731813348349 2024-11-17T03:15:51,658 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cb9242d2a43446baa8edcd745b4bfae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731813348370 2024-11-17T03:15:51,658 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412bafbbae3e4598983cdf56afb171c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731813350528 2024-11-17T03:15:51,661 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/26602e8cb6db45819fdb3259a5208f14 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26602e8cb6db45819fdb3259a5208f14 2024-11-17T03:15:51,670 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into 26602e8cb6db45819fdb3259a5208f14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:51,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:51,670 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=12, startTime=1731813351503; duration=0sec 2024-11-17T03:15:51,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:51,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:51,687 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:51,688 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/73c9a6e8f3c048d2a5a59e17271a4644 is 50, key is test_row_0/C:col10/1731813350534/Put/seqid=0 2024-11-17T03:15:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T03:15:51,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742107_1283 (size=13017) 2024-11-17T03:15:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:51,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:51,757 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/73c9a6e8f3c048d2a5a59e17271a4644 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c9a6e8f3c048d2a5a59e17271a4644 2024-11-17T03:15:51,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813411762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,769 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into 73c9a6e8f3c048d2a5a59e17271a4644(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:51,769 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:51,769 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=12, startTime=1731813351503; duration=0sec 2024-11-17T03:15:51,769 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:51,769 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:51,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813411765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813411767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813411767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813411768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813411877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813411887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813411887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813411887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,014 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/e34b34baa67b4706b1e51b42c085e829 2024-11-17T03:15:52,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/1c00e736989746bc85db51c6459ad270 is 50, key is test_row_0/B:col10/1731813350578/Put/seqid=0 2024-11-17T03:15:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813412087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742108_1284 (size=12301) 2024-11-17T03:15:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813412097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813412098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813412107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T03:15:52,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813412397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813412417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813412417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813412417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,494 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/1c00e736989746bc85db51c6459ad270 2024-11-17T03:15:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/5f05ab4128774c02a3d52e61dc9424e3 is 50, key is test_row_0/C:col10/1731813350578/Put/seqid=0 2024-11-17T03:15:52,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742109_1285 (size=12301) 2024-11-17T03:15:52,531 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/5f05ab4128774c02a3d52e61dc9424e3 2024-11-17T03:15:52,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/e34b34baa67b4706b1e51b42c085e829 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/e34b34baa67b4706b1e51b42c085e829 2024-11-17T03:15:52,544 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/e34b34baa67b4706b1e51b42c085e829, entries=150, sequenceid=559, filesize=12.0 K 2024-11-17T03:15:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/1c00e736989746bc85db51c6459ad270 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/1c00e736989746bc85db51c6459ad270 2024-11-17T03:15:52,553 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/1c00e736989746bc85db51c6459ad270, entries=150, sequenceid=559, filesize=12.0 K 2024-11-17T03:15:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/5f05ab4128774c02a3d52e61dc9424e3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/5f05ab4128774c02a3d52e61dc9424e3 2024-11-17T03:15:52,562 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/5f05ab4128774c02a3d52e61dc9424e3, entries=150, sequenceid=559, filesize=12.0 K 2024-11-17T03:15:52,563 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for a5d5a03df69af3fa2659ea9597853e20 in 1042ms, sequenceid=559, compaction requested=false 2024-11-17T03:15:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T03:15:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T03:15:52,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T03:15:52,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5100 sec 2024-11-17T03:15:52,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5190 sec 2024-11-17T03:15:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:52,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:15:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:52,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/0e95671cc28c438d9d17b30b3fa64398 is 50, key is test_row_0/A:col10/1731813351760/Put/seqid=0 2024-11-17T03:15:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742110_1286 (size=12301) 2024-11-17T03:15:52,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/0e95671cc28c438d9d17b30b3fa64398 2024-11-17T03:15:52,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/43366a566a924bcd8654122ec54b4ffc is 50, key is test_row_0/B:col10/1731813351760/Put/seqid=0 2024-11-17T03:15:52,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742111_1287 (size=12301) 2024-11-17T03:15:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813412863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813412907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813412926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813412926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813412926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813412977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:53,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813413187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T03:15:53,207 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:53,208 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:15:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T03:15:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T03:15:53,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:15:53,210 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:15:53,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:15:53,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/43366a566a924bcd8654122ec54b4ffc 2024-11-17T03:15:53,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/2650eb3143ad41a8bbb790c826e9fefc is 50, key is test_row_0/C:col10/1731813351760/Put/seqid=0 2024-11-17T03:15:53,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742112_1288 (size=12301) 2024-11-17T03:15:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T03:15:53,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T03:15:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:53,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813413497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:53,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T03:15:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:15:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T03:15:53,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/2650eb3143ad41a8bbb790c826e9fefc 2024-11-17T03:15:53,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/0e95671cc28c438d9d17b30b3fa64398 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0e95671cc28c438d9d17b30b3fa64398 2024-11-17T03:15:53,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0e95671cc28c438d9d17b30b3fa64398, entries=150, sequenceid=574, filesize=12.0 K 2024-11-17T03:15:53,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/43366a566a924bcd8654122ec54b4ffc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/43366a566a924bcd8654122ec54b4ffc 2024-11-17T03:15:53,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/43366a566a924bcd8654122ec54b4ffc, entries=150, sequenceid=574, filesize=12.0 K 2024-11-17T03:15:53,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/2650eb3143ad41a8bbb790c826e9fefc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2650eb3143ad41a8bbb790c826e9fefc 2024-11-17T03:15:53,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2650eb3143ad41a8bbb790c826e9fefc, entries=150, sequenceid=574, filesize=12.0 K 2024-11-17T03:15:53,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a5d5a03df69af3fa2659ea9597853e20 in 852ms, sequenceid=574, compaction requested=true 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:53,640 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:53,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:53,640 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:53,641 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:53,641 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:53,641 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/A is initiating minor compaction (all files) 2024-11-17T03:15:53,641 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/B is initiating minor compaction (all files) 2024-11-17T03:15:53,641 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/B in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,641 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/A in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,641 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26602e8cb6db45819fdb3259a5208f14, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/1c00e736989746bc85db51c6459ad270, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/43366a566a924bcd8654122ec54b4ffc] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=36.7 K 2024-11-17T03:15:53,641 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/7ebd29ca0250470f98ec941ec04d6aed, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/e34b34baa67b4706b1e51b42c085e829, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0e95671cc28c438d9d17b30b3fa64398] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=36.7 K 2024-11-17T03:15:53,641 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 26602e8cb6db45819fdb3259a5208f14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731813350528 2024-11-17T03:15:53,641 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ebd29ca0250470f98ec941ec04d6aed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731813350528 2024-11-17T03:15:53,642 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c00e736989746bc85db51c6459ad270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731813350578 2024-11-17T03:15:53,642 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e34b34baa67b4706b1e51b42c085e829, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731813350578 2024-11-17T03:15:53,642 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 43366a566a924bcd8654122ec54b4ffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731813351760 2024-11-17T03:15:53,642 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e95671cc28c438d9d17b30b3fa64398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731813351760 2024-11-17T03:15:53,649 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:53,649 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:53,649 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2fe2a39f2134434d9478a457ffb46e57 is 50, key is test_row_0/B:col10/1731813351760/Put/seqid=0 2024-11-17T03:15:53,649 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/65e702622f4846689fe7e8d04e23d3e5 is 50, key is test_row_0/A:col10/1731813351760/Put/seqid=0 2024-11-17T03:15:53,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742113_1289 (size=13119) 2024-11-17T03:15:53,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742114_1290 (size=13119) 2024-11-17T03:15:53,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T03:15:53,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,668 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T03:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:53,672 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/65e702622f4846689fe7e8d04e23d3e5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/65e702622f4846689fe7e8d04e23d3e5 2024-11-17T03:15:53,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b310886f1653447d89a7fe64b3c48ea6 is 50, key is test_row_0/A:col10/1731813352862/Put/seqid=0 2024-11-17T03:15:53,678 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/A of a5d5a03df69af3fa2659ea9597853e20 into 65e702622f4846689fe7e8d04e23d3e5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:53,678 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:53,678 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/A, priority=13, startTime=1731813353640; duration=0sec 2024-11-17T03:15:53,678 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:53,678 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:53,678 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:15:53,679 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:15:53,679 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): a5d5a03df69af3fa2659ea9597853e20/C is initiating minor compaction (all files) 2024-11-17T03:15:53,679 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5d5a03df69af3fa2659ea9597853e20/C in TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:53,679 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c9a6e8f3c048d2a5a59e17271a4644, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/5f05ab4128774c02a3d52e61dc9424e3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2650eb3143ad41a8bbb790c826e9fefc] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp, totalSize=36.7 K 2024-11-17T03:15:53,680 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c9a6e8f3c048d2a5a59e17271a4644, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1731813350528 2024-11-17T03:15:53,680 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f05ab4128774c02a3d52e61dc9424e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731813350578 2024-11-17T03:15:53,680 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2650eb3143ad41a8bbb790c826e9fefc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731813351760 2024-11-17T03:15:53,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742115_1291 (size=12301) 2024-11-17T03:15:53,689 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5d5a03df69af3fa2659ea9597853e20#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:15:53,689 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/e48375b02f744893ac190794424f2adc is 50, key is test_row_0/C:col10/1731813351760/Put/seqid=0 2024-11-17T03:15:53,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742116_1292 (size=13119) 2024-11-17T03:15:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T03:15:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. as already flushing 2024-11-17T03:15:53,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813413945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:53,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39052 deadline: 1731813413947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:53,949 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in a5d5a03df69af3fa2659ea9597853e20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:53,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39080 deadline: 1731813413947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:53,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813413947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:53,951 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in a5d5a03df69af3fa2659ea9597853e20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39088 deadline: 1731813414007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813414057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813414057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,065 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2fe2a39f2134434d9478a457ffb46e57 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2fe2a39f2134434d9478a457ffb46e57 2024-11-17T03:15:54,069 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/B of a5d5a03df69af3fa2659ea9597853e20 into 2fe2a39f2134434d9478a457ffb46e57(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:54,069 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:54,069 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/B, priority=13, startTime=1731813353640; duration=0sec 2024-11-17T03:15:54,069 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:54,069 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:54,081 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b310886f1653447d89a7fe64b3c48ea6 2024-11-17T03:15:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/522c271530f94069b0d78dd252a1e79e is 50, key is test_row_0/B:col10/1731813352862/Put/seqid=0 2024-11-17T03:15:54,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742117_1293 (size=12301) 2024-11-17T03:15:54,098 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/e48375b02f744893ac190794424f2adc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/e48375b02f744893ac190794424f2adc 2024-11-17T03:15:54,102 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5d5a03df69af3fa2659ea9597853e20/C of a5d5a03df69af3fa2659ea9597853e20 into e48375b02f744893ac190794424f2adc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:15:54,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:54,102 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., storeName=a5d5a03df69af3fa2659ea9597853e20/C, priority=13, startTime=1731813353640; duration=0sec 2024-11-17T03:15:54,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:54,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813414267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813414267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T03:15:54,352 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T03:15:54,492 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/522c271530f94069b0d78dd252a1e79e 2024-11-17T03:15:54,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/080ea3480c13470ab8b84e3e931438a6 is 50, key is test_row_0/C:col10/1731813352862/Put/seqid=0 2024-11-17T03:15:54,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742118_1294 (size=12301) 2024-11-17T03:15:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39124 deadline: 1731813414577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:15:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39140 deadline: 1731813414577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:15:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:15:54,909 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=594 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/080ea3480c13470ab8b84e3e931438a6 2024-11-17T03:15:54,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/b310886f1653447d89a7fe64b3c48ea6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b310886f1653447d89a7fe64b3c48ea6 2024-11-17T03:15:54,916 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b310886f1653447d89a7fe64b3c48ea6, entries=150, sequenceid=594, filesize=12.0 K 2024-11-17T03:15:54,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/522c271530f94069b0d78dd252a1e79e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/522c271530f94069b0d78dd252a1e79e 2024-11-17T03:15:54,923 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/522c271530f94069b0d78dd252a1e79e, entries=150, sequenceid=594, filesize=12.0 K 2024-11-17T03:15:54,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/080ea3480c13470ab8b84e3e931438a6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/080ea3480c13470ab8b84e3e931438a6 2024-11-17T03:15:54,928 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/080ea3480c13470ab8b84e3e931438a6, entries=150, sequenceid=594, filesize=12.0 K 2024-11-17T03:15:54,928 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a5d5a03df69af3fa2659ea9597853e20 in 1260ms, sequenceid=594, compaction requested=false 2024-11-17T03:15:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T03:15:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T03:15:54,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T03:15:54,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-17T03:15:54,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7250 sec 2024-11-17T03:15:55,004 INFO [Thread-1038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1038. 2024-11-17T03:15:55,004 INFO [Thread-1042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1042. 2024-11-17T03:15:55,004 DEBUG [Thread-1042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,004 DEBUG [Thread-1038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,004 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,004 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,004 DEBUG [Thread-1038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,004 INFO [Thread-1034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1034. 2024-11-17T03:15:55,004 DEBUG [Thread-1034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,004 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,004 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,005 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,006 INFO [Thread-1036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1036. 2024-11-17T03:15:55,006 DEBUG [Thread-1036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,006 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-11-17T03:15:55,006 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,006 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,006 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,006 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,006 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:55,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T03:15:55,029 INFO [Thread-1025 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1025. 2024-11-17T03:15:55,029 DEBUG [Thread-1025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,029 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:55,029 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:55,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:55,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:55,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/37c7e39bcb14487aa26adccfd43ac466 is 50, key is test_row_0/A:col10/1731813353944/Put/seqid=0 2024-11-17T03:15:55,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742119_1295 (size=12301) 2024-11-17T03:15:55,089 INFO [Thread-1029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1029. 2024-11-17T03:15:55,089 INFO [Thread-1023 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1023. 2024-11-17T03:15:55,089 DEBUG [Thread-1029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,089 DEBUG [Thread-1023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,089 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,089 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,089 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,089 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T03:15:55,358 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:55,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/37c7e39bcb14487aa26adccfd43ac466 2024-11-17T03:15:55,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/84f3fc93ec374eacb76b76afbee9a4e2 is 50, key is test_row_0/B:col10/1731813353944/Put/seqid=0 2024-11-17T03:15:55,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742120_1296 (size=12301) 2024-11-17T03:15:55,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/84f3fc93ec374eacb76b76afbee9a4e2 2024-11-17T03:15:55,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/00ef81bf19b74e25a998726ee61caaee is 50, key is test_row_0/C:col10/1731813353944/Put/seqid=0 2024-11-17T03:15:55,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742121_1297 (size=12301) 2024-11-17T03:15:55,959 INFO [Thread-1031 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1031. 2024-11-17T03:15:55,959 DEBUG [Thread-1031 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,959 DEBUG [Thread-1031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,959 DEBUG [Thread-1031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,969 INFO [Thread-1027 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1027. 2024-11-17T03:15:55,969 DEBUG [Thread-1027 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:15:55,969 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,969 DEBUG [Thread-1027 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,969 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2889 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2974 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2828 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2873 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2966 2024-11-17T03:15:55,970 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T03:15:55,970 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:15:55,970 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:15:55,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:15:55,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:15:55,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T03:15:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T03:15:55,977 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813355977"}]},"ts":"1731813355977"} 2024-11-17T03:15:55,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T03:15:55,980 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T03:15:55,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:15:55,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, UNASSIGN}] 2024-11-17T03:15:55,983 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, UNASSIGN 2024-11-17T03:15:55,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a5d5a03df69af3fa2659ea9597853e20, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:55,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, UNASSIGN because future has completed 2024-11-17T03:15:55,985 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:15:55,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T03:15:56,138 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:56,138 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:15:56,138 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing a5d5a03df69af3fa2659ea9597853e20, disabling compactions & flushes 2024-11-17T03:15:56,138 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T03:15:56,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/00ef81bf19b74e25a998726ee61caaee 2024-11-17T03:15:56,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/37c7e39bcb14487aa26adccfd43ac466 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/37c7e39bcb14487aa26adccfd43ac466 2024-11-17T03:15:56,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/37c7e39bcb14487aa26adccfd43ac466, entries=150, sequenceid=610, filesize=12.0 K 2024-11-17T03:15:56,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/84f3fc93ec374eacb76b76afbee9a4e2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/84f3fc93ec374eacb76b76afbee9a4e2 2024-11-17T03:15:56,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/84f3fc93ec374eacb76b76afbee9a4e2, entries=150, sequenceid=610, filesize=12.0 K 2024-11-17T03:15:56,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/00ef81bf19b74e25a998726ee61caaee as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/00ef81bf19b74e25a998726ee61caaee 2024-11-17T03:15:56,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/00ef81bf19b74e25a998726ee61caaee, entries=150, sequenceid=610, filesize=12.0 K 2024-11-17T03:15:56,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for a5d5a03df69af3fa2659ea9597853e20 in 1319ms, sequenceid=610, compaction requested=true 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5d5a03df69af3fa2659ea9597853e20: 2024-11-17T03:15:56,348 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:56,348 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:15:56,348 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. after waiting 0 ms 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:56,348 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:15:56,348 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. because compaction request was cancelled 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5d5a03df69af3fa2659ea9597853e20:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:15:56,348 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing a5d5a03df69af3fa2659ea9597853e20 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T03:15:56,348 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. because compaction request was cancelled 2024-11-17T03:15:56,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:15:56,348 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:A 2024-11-17T03:15:56,348 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:B 2024-11-17T03:15:56,348 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. because compaction request was cancelled 2024-11-17T03:15:56,348 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5d5a03df69af3fa2659ea9597853e20:C 2024-11-17T03:15:56,348 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=A 2024-11-17T03:15:56,348 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:56,349 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=B 2024-11-17T03:15:56,349 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:56,349 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5d5a03df69af3fa2659ea9597853e20, store=C 2024-11-17T03:15:56,349 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:15:56,352 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/628d37e50e8148aa9c7bac83b4d41b33 is 50, key is test_row_0/A:col10/1731813355967/Put/seqid=0 2024-11-17T03:15:56,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742122_1298 (size=9857) 2024-11-17T03:15:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T03:15:56,759 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/628d37e50e8148aa9c7bac83b4d41b33 2024-11-17T03:15:56,765 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2afeb607a11f411cbcc639cd20dbe32c is 50, key is test_row_0/B:col10/1731813355967/Put/seqid=0 2024-11-17T03:15:56,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742123_1299 (size=9857) 2024-11-17T03:15:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T03:15:57,170 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2afeb607a11f411cbcc639cd20dbe32c 2024-11-17T03:15:57,180 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4c203443281445a69343b04fdbffeb90 is 50, key is test_row_0/C:col10/1731813355967/Put/seqid=0 2024-11-17T03:15:57,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742124_1300 (size=9857) 2024-11-17T03:15:57,584 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4c203443281445a69343b04fdbffeb90 2024-11-17T03:15:57,590 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/A/628d37e50e8148aa9c7bac83b4d41b33 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/628d37e50e8148aa9c7bac83b4d41b33 2024-11-17T03:15:57,596 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/628d37e50e8148aa9c7bac83b4d41b33, entries=100, sequenceid=617, filesize=9.6 K 2024-11-17T03:15:57,598 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/B/2afeb607a11f411cbcc639cd20dbe32c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2afeb607a11f411cbcc639cd20dbe32c 2024-11-17T03:15:57,602 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2afeb607a11f411cbcc639cd20dbe32c, entries=100, sequenceid=617, filesize=9.6 K 2024-11-17T03:15:57,603 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/.tmp/C/4c203443281445a69343b04fdbffeb90 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4c203443281445a69343b04fdbffeb90 2024-11-17T03:15:57,606 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4c203443281445a69343b04fdbffeb90, entries=100, sequenceid=617, filesize=9.6 K 2024-11-17T03:15:57,607 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a5d5a03df69af3fa2659ea9597853e20 in 1259ms, sequenceid=617, compaction requested=true 2024-11-17T03:15:57,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b64216f0c9bc4bdc83c443ce2c8d3039, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/05ed1e9761994b03bd2219ef14927759, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/5502a4e1dbcf459dabcb570216837359, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/6ac99f08825d4494b4f999dbaa7d8307, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b1d9e588e27241db880c9c73e15c53e6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/4177082470654b1189f05f79ab427946, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/355f3254157742c6a79fe21a2cd8beb1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/904aa740549040ee91101201a23045e5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/cc7454b21db249f7ad405135f59c2db0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/11c461a30a79416e86a3f09a6e61b1a1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/3a4c6f811bd94e52a1f91bc9004ae6ee, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d4d3532e94ac4a658c040d055675ee15, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/89bd9f8d7809428baf20873c806228e2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/c919904f09b144b4be3c0d8048f341df, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/796fc5f78c414c20934a7a26df0c92fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/8f538ad6cbc54ab0bc574ba101521ba5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/48cd7f6b4d784a9997594f3d84d75fc2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d9e5097bc09940a896fab75ded634ff7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/955aee62d8e746d08d445e50a904fdfe, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/fbe1850d75644e55b268d6b3ecaf90da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0048f5707a864e9e8a50f915e0f1005e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/7ebd29ca0250470f98ec941ec04d6aed, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/e34b34baa67b4706b1e51b42c085e829, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0e95671cc28c438d9d17b30b3fa64398] to archive 2024-11-17T03:15:57,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:57,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b64216f0c9bc4bdc83c443ce2c8d3039 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b64216f0c9bc4bdc83c443ce2c8d3039 2024-11-17T03:15:57,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/05ed1e9761994b03bd2219ef14927759 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/05ed1e9761994b03bd2219ef14927759 2024-11-17T03:15:57,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/5502a4e1dbcf459dabcb570216837359 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/5502a4e1dbcf459dabcb570216837359 2024-11-17T03:15:57,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/6ac99f08825d4494b4f999dbaa7d8307 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/6ac99f08825d4494b4f999dbaa7d8307 2024-11-17T03:15:57,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b1d9e588e27241db880c9c73e15c53e6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b1d9e588e27241db880c9c73e15c53e6 2024-11-17T03:15:57,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/4177082470654b1189f05f79ab427946 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/4177082470654b1189f05f79ab427946 2024-11-17T03:15:57,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/355f3254157742c6a79fe21a2cd8beb1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/355f3254157742c6a79fe21a2cd8beb1 2024-11-17T03:15:57,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/904aa740549040ee91101201a23045e5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/904aa740549040ee91101201a23045e5 2024-11-17T03:15:57,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/cc7454b21db249f7ad405135f59c2db0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/cc7454b21db249f7ad405135f59c2db0 2024-11-17T03:15:57,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/11c461a30a79416e86a3f09a6e61b1a1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/11c461a30a79416e86a3f09a6e61b1a1 2024-11-17T03:15:57,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/3a4c6f811bd94e52a1f91bc9004ae6ee to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/3a4c6f811bd94e52a1f91bc9004ae6ee 2024-11-17T03:15:57,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d4d3532e94ac4a658c040d055675ee15 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d4d3532e94ac4a658c040d055675ee15 2024-11-17T03:15:57,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/89bd9f8d7809428baf20873c806228e2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/89bd9f8d7809428baf20873c806228e2 2024-11-17T03:15:57,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/c919904f09b144b4be3c0d8048f341df to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/c919904f09b144b4be3c0d8048f341df 2024-11-17T03:15:57,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/796fc5f78c414c20934a7a26df0c92fd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/796fc5f78c414c20934a7a26df0c92fd 2024-11-17T03:15:57,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/8f538ad6cbc54ab0bc574ba101521ba5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/8f538ad6cbc54ab0bc574ba101521ba5 2024-11-17T03:15:57,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/48cd7f6b4d784a9997594f3d84d75fc2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/48cd7f6b4d784a9997594f3d84d75fc2 2024-11-17T03:15:57,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d9e5097bc09940a896fab75ded634ff7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/d9e5097bc09940a896fab75ded634ff7 2024-11-17T03:15:57,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/955aee62d8e746d08d445e50a904fdfe to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/955aee62d8e746d08d445e50a904fdfe 2024-11-17T03:15:57,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/fbe1850d75644e55b268d6b3ecaf90da to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/fbe1850d75644e55b268d6b3ecaf90da 2024-11-17T03:15:57,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0048f5707a864e9e8a50f915e0f1005e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0048f5707a864e9e8a50f915e0f1005e 2024-11-17T03:15:57,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/7ebd29ca0250470f98ec941ec04d6aed to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/7ebd29ca0250470f98ec941ec04d6aed 2024-11-17T03:15:57,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/e34b34baa67b4706b1e51b42c085e829 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/e34b34baa67b4706b1e51b42c085e829 2024-11-17T03:15:57,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0e95671cc28c438d9d17b30b3fa64398 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/0e95671cc28c438d9d17b30b3fa64398 2024-11-17T03:15:57,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/b43abf8653e845f1b12a7e4652426742, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/3e020b90ecb04a08ad9cc8ea2b328f8a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/51d2594fd569432e8bf9e73ecfd840b0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/87d074227341471292b3bf497ace0502, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4fba7169049643a3a5dd4a1ab0ff6ee9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/9820e21f8d034324bf0cc56364bb6d6c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4190b1df2e7c4b27886d155134c8ed1b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2b200a29d8794ea7882b50950e0719ae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8b4fe41ec4764f9094c4ce79a4795f50, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8460fe1762ed4637b2f3338b8ecc3a0e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/ee6dc6498a8c428fa91ef1ad56781cc5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4d1c68e712644946931a4b46d716c6a1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/afbdc5b9d9184c5a8f7c7c140eb849d5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/eb637c0ff0904d8b9251d932b145936c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26a3cc992ec94cbea81507439449f6a6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/0a32b275bc764569a82bb896eabcc0e6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c93b66dc706848ef986ecfc622944cee, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c08ac03526b943bf901484ddee3d1214, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/5081ed4a75404e66b1688235022eca78, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/d31f0097f98c47919369e4a41c455d2d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26602e8cb6db45819fdb3259a5208f14, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/96cce80083804a0c8276f861c05f62bc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/1c00e736989746bc85db51c6459ad270, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/43366a566a924bcd8654122ec54b4ffc] to archive 2024-11-17T03:15:57,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:57,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/b43abf8653e845f1b12a7e4652426742 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/b43abf8653e845f1b12a7e4652426742 2024-11-17T03:15:57,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/3e020b90ecb04a08ad9cc8ea2b328f8a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/3e020b90ecb04a08ad9cc8ea2b328f8a 2024-11-17T03:15:57,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/51d2594fd569432e8bf9e73ecfd840b0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/51d2594fd569432e8bf9e73ecfd840b0 2024-11-17T03:15:57,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/87d074227341471292b3bf497ace0502 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/87d074227341471292b3bf497ace0502 2024-11-17T03:15:57,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4fba7169049643a3a5dd4a1ab0ff6ee9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4fba7169049643a3a5dd4a1ab0ff6ee9 2024-11-17T03:15:57,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/9820e21f8d034324bf0cc56364bb6d6c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/9820e21f8d034324bf0cc56364bb6d6c 2024-11-17T03:15:57,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4190b1df2e7c4b27886d155134c8ed1b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4190b1df2e7c4b27886d155134c8ed1b 2024-11-17T03:15:57,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2b200a29d8794ea7882b50950e0719ae to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2b200a29d8794ea7882b50950e0719ae 2024-11-17T03:15:57,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8b4fe41ec4764f9094c4ce79a4795f50 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8b4fe41ec4764f9094c4ce79a4795f50 2024-11-17T03:15:57,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8460fe1762ed4637b2f3338b8ecc3a0e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/8460fe1762ed4637b2f3338b8ecc3a0e 2024-11-17T03:15:57,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/ee6dc6498a8c428fa91ef1ad56781cc5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/ee6dc6498a8c428fa91ef1ad56781cc5 2024-11-17T03:15:57,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4d1c68e712644946931a4b46d716c6a1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/4d1c68e712644946931a4b46d716c6a1 2024-11-17T03:15:57,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/afbdc5b9d9184c5a8f7c7c140eb849d5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/afbdc5b9d9184c5a8f7c7c140eb849d5 2024-11-17T03:15:57,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/eb637c0ff0904d8b9251d932b145936c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/eb637c0ff0904d8b9251d932b145936c 2024-11-17T03:15:57,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26a3cc992ec94cbea81507439449f6a6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26a3cc992ec94cbea81507439449f6a6 2024-11-17T03:15:57,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/0a32b275bc764569a82bb896eabcc0e6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/0a32b275bc764569a82bb896eabcc0e6 2024-11-17T03:15:57,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c93b66dc706848ef986ecfc622944cee to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c93b66dc706848ef986ecfc622944cee 2024-11-17T03:15:57,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c08ac03526b943bf901484ddee3d1214 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/c08ac03526b943bf901484ddee3d1214 2024-11-17T03:15:57,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/5081ed4a75404e66b1688235022eca78 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/5081ed4a75404e66b1688235022eca78 2024-11-17T03:15:57,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/d31f0097f98c47919369e4a41c455d2d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/d31f0097f98c47919369e4a41c455d2d 2024-11-17T03:15:57,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26602e8cb6db45819fdb3259a5208f14 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/26602e8cb6db45819fdb3259a5208f14 2024-11-17T03:15:57,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/96cce80083804a0c8276f861c05f62bc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/96cce80083804a0c8276f861c05f62bc 2024-11-17T03:15:57,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/1c00e736989746bc85db51c6459ad270 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/1c00e736989746bc85db51c6459ad270 2024-11-17T03:15:57,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/43366a566a924bcd8654122ec54b4ffc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/43366a566a924bcd8654122ec54b4ffc 2024-11-17T03:15:57,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c2cbf8c7144a108a59a44c8ffb8e74, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/dc8b028ba26c40d09724ba132ba8e88f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/540832c5e5904ac4901ac52a795bd6c8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/1aa380a16c8647dbb2741ebc95571125, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/de9b8c1aa14b4e9d82d72382e8a7ccf9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/b6b972c687cf434d80d831a58e03befa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/07dfdf0ef439434eadb164d8be066e17, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4fea6e36ea37482f8a62b77be8f76258, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4f82fa80f65f485eb83f3ab43ed151f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4040a43112174094a839680c6e1f5aad, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/9ba9cf5ec0b54859aa04b09cf89b62b8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2b6e527627c7427d85cc74c832f49ba8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/7555c0393c1646b692e2cc0de41d5d14, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/069d6a8887e84be09d982eacdf028930, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8379b3a411814e939a4a7ed29187da25, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/bc3a33d331e049c3b08fa3ae6b15b4cf, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/18abfacf7f8f4d7d8760cdf645b67f05, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/83a5f2f7d1e84598b5ed548fd61cef8d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/547121878b834ef088e0099d4f3aa285, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8cb9242d2a43446baa8edcd745b4bfae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c9a6e8f3c048d2a5a59e17271a4644, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/412bafbbae3e4598983cdf56afb171c6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/5f05ab4128774c02a3d52e61dc9424e3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2650eb3143ad41a8bbb790c826e9fefc] to archive 2024-11-17T03:15:57,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:15:57,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c2cbf8c7144a108a59a44c8ffb8e74 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c2cbf8c7144a108a59a44c8ffb8e74 2024-11-17T03:15:57,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/dc8b028ba26c40d09724ba132ba8e88f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/dc8b028ba26c40d09724ba132ba8e88f 2024-11-17T03:15:57,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/540832c5e5904ac4901ac52a795bd6c8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/540832c5e5904ac4901ac52a795bd6c8 2024-11-17T03:15:57,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/1aa380a16c8647dbb2741ebc95571125 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/1aa380a16c8647dbb2741ebc95571125 2024-11-17T03:15:57,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/de9b8c1aa14b4e9d82d72382e8a7ccf9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/de9b8c1aa14b4e9d82d72382e8a7ccf9 2024-11-17T03:15:57,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/b6b972c687cf434d80d831a58e03befa to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/b6b972c687cf434d80d831a58e03befa 2024-11-17T03:15:57,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/07dfdf0ef439434eadb164d8be066e17 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/07dfdf0ef439434eadb164d8be066e17 2024-11-17T03:15:57,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4fea6e36ea37482f8a62b77be8f76258 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4fea6e36ea37482f8a62b77be8f76258 2024-11-17T03:15:57,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4f82fa80f65f485eb83f3ab43ed151f1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4f82fa80f65f485eb83f3ab43ed151f1 2024-11-17T03:15:57,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4040a43112174094a839680c6e1f5aad to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4040a43112174094a839680c6e1f5aad 2024-11-17T03:15:57,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/9ba9cf5ec0b54859aa04b09cf89b62b8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/9ba9cf5ec0b54859aa04b09cf89b62b8 2024-11-17T03:15:57,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2b6e527627c7427d85cc74c832f49ba8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2b6e527627c7427d85cc74c832f49ba8 2024-11-17T03:15:57,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/7555c0393c1646b692e2cc0de41d5d14 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/7555c0393c1646b692e2cc0de41d5d14 2024-11-17T03:15:57,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/069d6a8887e84be09d982eacdf028930 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/069d6a8887e84be09d982eacdf028930 2024-11-17T03:15:57,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8379b3a411814e939a4a7ed29187da25 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8379b3a411814e939a4a7ed29187da25 2024-11-17T03:15:57,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/bc3a33d331e049c3b08fa3ae6b15b4cf to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/bc3a33d331e049c3b08fa3ae6b15b4cf 2024-11-17T03:15:57,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/18abfacf7f8f4d7d8760cdf645b67f05 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/18abfacf7f8f4d7d8760cdf645b67f05 2024-11-17T03:15:57,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/83a5f2f7d1e84598b5ed548fd61cef8d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/83a5f2f7d1e84598b5ed548fd61cef8d 2024-11-17T03:15:57,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/547121878b834ef088e0099d4f3aa285 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/547121878b834ef088e0099d4f3aa285 2024-11-17T03:15:57,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8cb9242d2a43446baa8edcd745b4bfae to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/8cb9242d2a43446baa8edcd745b4bfae 2024-11-17T03:15:57,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c9a6e8f3c048d2a5a59e17271a4644 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/73c9a6e8f3c048d2a5a59e17271a4644 2024-11-17T03:15:57,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/412bafbbae3e4598983cdf56afb171c6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/412bafbbae3e4598983cdf56afb171c6 2024-11-17T03:15:57,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/5f05ab4128774c02a3d52e61dc9424e3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/5f05ab4128774c02a3d52e61dc9424e3 2024-11-17T03:15:57,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2650eb3143ad41a8bbb790c826e9fefc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/2650eb3143ad41a8bbb790c826e9fefc 2024-11-17T03:15:57,683 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/recovered.edits/620.seqid, newMaxSeqId=620, maxSeqId=1 2024-11-17T03:15:57,684 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. 2024-11-17T03:15:57,684 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for a5d5a03df69af3fa2659ea9597853e20: Waiting for close lock at 1731813356138Running coprocessor pre-close hooks at 1731813356138Disabling compacts and flushes for region at 1731813356138Disabling writes for close at 1731813356348 (+210 ms)Obtaining lock to block concurrent updates at 1731813356348Preparing flush snapshotting stores in a5d5a03df69af3fa2659ea9597853e20 at 1731813356348Finished memstore snapshotting TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731813356349 (+1 ms)Flushing stores of TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20. at 1731813356349Flushing a5d5a03df69af3fa2659ea9597853e20/A: creating writer at 1731813356349Flushing a5d5a03df69af3fa2659ea9597853e20/A: appending metadata at 1731813356352 (+3 ms)Flushing a5d5a03df69af3fa2659ea9597853e20/A: closing flushed file at 1731813356352Flushing a5d5a03df69af3fa2659ea9597853e20/B: creating writer at 1731813356763 (+411 ms)Flushing a5d5a03df69af3fa2659ea9597853e20/B: appending metadata at 1731813356765 (+2 ms)Flushing a5d5a03df69af3fa2659ea9597853e20/B: closing flushed file at 1731813356765Flushing a5d5a03df69af3fa2659ea9597853e20/C: creating writer at 1731813357176 (+411 ms)Flushing a5d5a03df69af3fa2659ea9597853e20/C: appending metadata at 1731813357179 (+3 ms)Flushing a5d5a03df69af3fa2659ea9597853e20/C: closing flushed file at 1731813357179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d9c6332: reopening flushed file at 1731813357589 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16d2e297: reopening flushed file at 1731813357597 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23910959: reopening flushed file at 1731813357602 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a5d5a03df69af3fa2659ea9597853e20 in 1259ms, sequenceid=617, compaction requested=true at 1731813357607 (+5 ms)Writing region close event to WAL at 1731813357680 (+73 ms)Running coprocessor post-close hooks at 1731813357684 (+4 ms)Closed at 1731813357684 2024-11-17T03:15:57,686 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:57,686 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a5d5a03df69af3fa2659ea9597853e20, regionState=CLOSED 2024-11-17T03:15:57,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:57,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T03:15:57,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure a5d5a03df69af3fa2659ea9597853e20, server=695c90f4bb18,38753,1731813266381 in 1.7030 sec 2024-11-17T03:15:57,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-17T03:15:57,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5d5a03df69af3fa2659ea9597853e20, UNASSIGN in 1.7070 sec 2024-11-17T03:15:57,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T03:15:57,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7110 sec 2024-11-17T03:15:57,692 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813357692"}]},"ts":"1731813357692"} 2024-11-17T03:15:57,693 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T03:15:57,693 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T03:15:57,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7220 sec 2024-11-17T03:15:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T03:15:58,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:58,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T03:15:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,130 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T03:15:58,130 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,132 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:58,134 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/recovered.edits] 2024-11-17T03:15:58,136 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/37c7e39bcb14487aa26adccfd43ac466 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/37c7e39bcb14487aa26adccfd43ac466 2024-11-17T03:15:58,137 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/628d37e50e8148aa9c7bac83b4d41b33 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/628d37e50e8148aa9c7bac83b4d41b33 2024-11-17T03:15:58,138 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/65e702622f4846689fe7e8d04e23d3e5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/65e702622f4846689fe7e8d04e23d3e5 2024-11-17T03:15:58,139 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b310886f1653447d89a7fe64b3c48ea6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/A/b310886f1653447d89a7fe64b3c48ea6 2024-11-17T03:15:58,141 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2afeb607a11f411cbcc639cd20dbe32c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2afeb607a11f411cbcc639cd20dbe32c 2024-11-17T03:15:58,142 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2fe2a39f2134434d9478a457ffb46e57 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/2fe2a39f2134434d9478a457ffb46e57 2024-11-17T03:15:58,143 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/522c271530f94069b0d78dd252a1e79e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/522c271530f94069b0d78dd252a1e79e 2024-11-17T03:15:58,144 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/84f3fc93ec374eacb76b76afbee9a4e2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/B/84f3fc93ec374eacb76b76afbee9a4e2 2024-11-17T03:15:58,146 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/00ef81bf19b74e25a998726ee61caaee to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/00ef81bf19b74e25a998726ee61caaee 2024-11-17T03:15:58,147 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/080ea3480c13470ab8b84e3e931438a6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/080ea3480c13470ab8b84e3e931438a6 2024-11-17T03:15:58,147 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4c203443281445a69343b04fdbffeb90 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/4c203443281445a69343b04fdbffeb90 2024-11-17T03:15:58,148 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/e48375b02f744893ac190794424f2adc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/C/e48375b02f744893ac190794424f2adc 2024-11-17T03:15:58,150 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/recovered.edits/620.seqid to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20/recovered.edits/620.seqid 2024-11-17T03:15:58,150 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/a5d5a03df69af3fa2659ea9597853e20 2024-11-17T03:15:58,150 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T03:15:58,152 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,154 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T03:15:58,156 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T03:15:58,157 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,157 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T03:15:58,157 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731813358157"}]},"ts":"9223372036854775807"} 2024-11-17T03:15:58,159 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T03:15:58,159 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a5d5a03df69af3fa2659ea9597853e20, NAME => 'TestAcidGuarantees,,1731813332759.a5d5a03df69af3fa2659ea9597853e20.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T03:15:58,159 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T03:15:58,159 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731813358159"}]},"ts":"9223372036854775807"} 2024-11-17T03:15:58,161 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T03:15:58,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-17T03:15:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T03:15:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T03:15:58,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:15:58,247 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 251), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 412) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1655 (was 2569) 2024-11-17T03:15:58,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=467, ProcessCount=11, AvailableMemoryMB=1655 2024-11-17T03:15:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:15:58,256 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:15:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T03:15:58,258 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T03:15:58,258 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:58,258 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-17T03:15:58,259 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T03:15:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T03:15:58,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742125_1301 (size=1061) 2024-11-17T03:15:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T03:15:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T03:15:58,667 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:15:58,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742126_1302 (size=53) 2024-11-17T03:15:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T03:15:59,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:59,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aa45336f99df582886fbc013393f2fd9, disabling compactions & flushes 2024-11-17T03:15:59,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. after waiting 0 ms 2024-11-17T03:15:59,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,077 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa45336f99df582886fbc013393f2fd9: Waiting for close lock at 1731813359076Disabling compacts and flushes for region at 1731813359076Disabling writes for close at 1731813359077 (+1 ms)Writing region close event to WAL at 1731813359077Closed at 1731813359077 2024-11-17T03:15:59,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T03:15:59,081 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731813359080"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731813359080"}]},"ts":"1731813359080"} 2024-11-17T03:15:59,083 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T03:15:59,084 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T03:15:59,085 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813359084"}]},"ts":"1731813359084"} 2024-11-17T03:15:59,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T03:15:59,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, ASSIGN}] 2024-11-17T03:15:59,088 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, ASSIGN 2024-11-17T03:15:59,090 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, ASSIGN; state=OFFLINE, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=false 2024-11-17T03:15:59,241 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:59,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, ASSIGN because future has completed 2024-11-17T03:15:59,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T03:15:59,399 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,399 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:15:59,400 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,400 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:15:59,400 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,400 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,402 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,404 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:59,404 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa45336f99df582886fbc013393f2fd9 columnFamilyName A 2024-11-17T03:15:59,404 DEBUG [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:59,405 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(327): Store=aa45336f99df582886fbc013393f2fd9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:59,405 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,407 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:59,407 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa45336f99df582886fbc013393f2fd9 columnFamilyName B 2024-11-17T03:15:59,407 DEBUG [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:59,408 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(327): Store=aa45336f99df582886fbc013393f2fd9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:59,408 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,409 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:15:59,409 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa45336f99df582886fbc013393f2fd9 columnFamilyName C 2024-11-17T03:15:59,409 DEBUG [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:15:59,410 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(327): Store=aa45336f99df582886fbc013393f2fd9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:15:59,410 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,410 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,411 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,412 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,413 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,413 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,414 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:15:59,416 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,418 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:15:59,419 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened aa45336f99df582886fbc013393f2fd9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71210973, jitterRate=0.061126187443733215}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:15:59,419 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:15:59,420 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for aa45336f99df582886fbc013393f2fd9: Running coprocessor pre-open hook at 1731813359400Writing region info on filesystem at 1731813359400Initializing all the Stores at 1731813359401 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813359402 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813359402Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813359402Cleaning up temporary data from old regions at 1731813359413 (+11 ms)Running coprocessor post-open hooks at 1731813359419 (+6 ms)Region opened successfully at 1731813359420 (+1 ms) 2024-11-17T03:15:59,421 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., pid=87, masterSystemTime=1731813359395 2024-11-17T03:15:59,423 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,423 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:15:59,424 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:15:59,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:15:59,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T03:15:59,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 in 183 msec 2024-11-17T03:15:59,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-17T03:15:59,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, ASSIGN in 341 msec 2024-11-17T03:15:59,431 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T03:15:59,431 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813359431"}]},"ts":"1731813359431"} 2024-11-17T03:15:59,432 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T03:15:59,433 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T03:15:59,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-17T03:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T03:16:00,407 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:00,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c48710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:00,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:00,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:00,409 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb6fd7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:00,411 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40bd34a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:00,412 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:00,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:00,415 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:16:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:16:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fd1cedc 2024-11-17T03:16:00,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:16:00,417 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:16:00,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:16:00,419 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:16:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:00,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742127_1303 (size=1097) 2024-11-17T03:16:00,834 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T03:16:00,834 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T03:16:00,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:16:00,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, REOPEN/MOVE}] 2024-11-17T03:16:00,841 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, REOPEN/MOVE 2024-11-17T03:16:00,842 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:00,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, REOPEN/MOVE because future has completed 2024-11-17T03:16:00,843 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:16:00,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:00,997 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:00,997 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:16:00,997 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing aa45336f99df582886fbc013393f2fd9, disabling compactions & flushes 2024-11-17T03:16:00,997 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:00,997 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:00,997 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. after waiting 0 ms 2024-11-17T03:16:00,997 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,000 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T03:16:01,001 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,001 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for aa45336f99df582886fbc013393f2fd9: Waiting for close lock at 1731813360997Running coprocessor pre-close hooks at 1731813360997Disabling compacts and flushes for region at 1731813360997Disabling writes for close at 1731813360997Writing region close event to WAL at 1731813360997Running coprocessor post-close hooks at 1731813361001 (+4 ms)Closed at 1731813361001 2024-11-17T03:16:01,001 WARN [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: aa45336f99df582886fbc013393f2fd9 to self. 2024-11-17T03:16:01,002 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,003 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=CLOSED 2024-11-17T03:16:01,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:01,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T03:16:01,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 in 162 msec 2024-11-17T03:16:01,007 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, REOPEN/MOVE; state=CLOSED, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=true 2024-11-17T03:16:01,158 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, REOPEN/MOVE because future has completed 2024-11-17T03:16:01,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:01,317 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,317 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:16:01,317 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,317 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:16:01,317 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,318 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,319 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,320 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:01,320 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa45336f99df582886fbc013393f2fd9 columnFamilyName A 2024-11-17T03:16:01,321 DEBUG [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:01,321 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(327): Store=aa45336f99df582886fbc013393f2fd9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:01,321 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,322 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:01,322 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa45336f99df582886fbc013393f2fd9 columnFamilyName B 2024-11-17T03:16:01,322 DEBUG [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:01,323 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(327): Store=aa45336f99df582886fbc013393f2fd9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:01,323 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,323 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:01,323 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa45336f99df582886fbc013393f2fd9 columnFamilyName C 2024-11-17T03:16:01,323 DEBUG [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:01,324 INFO [StoreOpener-aa45336f99df582886fbc013393f2fd9-1 {}] regionserver.HStore(327): Store=aa45336f99df582886fbc013393f2fd9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:01,324 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,324 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,325 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,326 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,327 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,327 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,327 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:16:01,328 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,329 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened aa45336f99df582886fbc013393f2fd9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70556244, jitterRate=0.05136996507644653}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:16:01,329 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,330 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for aa45336f99df582886fbc013393f2fd9: Running coprocessor pre-open hook at 1731813361318Writing region info on filesystem at 1731813361318Initializing all the Stores at 1731813361319 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813361319Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813361319Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813361319Cleaning up temporary data from old regions at 1731813361327 (+8 ms)Running coprocessor post-open hooks at 1731813361329 (+2 ms)Region opened successfully at 1731813361329 2024-11-17T03:16:01,330 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., pid=92, masterSystemTime=1731813361311 2024-11-17T03:16:01,332 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,332 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,332 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=OPEN, openSeqNum=5, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:01,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-17T03:16:01,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 in 175 msec 2024-11-17T03:16:01,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-17T03:16:01,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, REOPEN/MOVE in 495 msec 2024-11-17T03:16:01,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T03:16:01,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-17T03:16:01,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 919 msec 2024-11-17T03:16:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T03:16:01,342 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:01,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a23409f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,342 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,343 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2792f786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,344 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e399747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50111e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,346 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed4a5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@693ad426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144ae751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,349 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c697f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,350 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c46ac7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1663b052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,351 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06fab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,352 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738f8f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c8a0ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,353 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c88de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b140cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351827da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,356 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3113ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,357 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426cd0a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30be4484, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f271be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,360 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d404327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e17d831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,362 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd11282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,363 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d14fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cadfc7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,365 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cd3155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,367 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692b5a2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5708b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:01,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:01,369 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3cd87e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:01,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:01,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b9aa7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:01,371 DEBUG [Thread-1401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,371 DEBUG [Thread-1418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,371 DEBUG [Thread-1414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,371 DEBUG [Thread-1403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,371 DEBUG [Thread-1405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,371 DEBUG [Thread-1410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,372 DEBUG [Thread-1416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,372 DEBUG [Thread-1405 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,373 DEBUG [Thread-1405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,373 DEBUG [Thread-1407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,374 DEBUG [Thread-1399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:01,376 DEBUG [Thread-1412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:01,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-17T03:16:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:01,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:01,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:01,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:01,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:16:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:01,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117de305d8165cf485982fb60580ecbf02c_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813361387/Put/seqid=0 2024-11-17T03:16:01,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742128_1304 (size=14594) 2024-11-17T03:16:01,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:01,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117de305d8165cf485982fb60580ecbf02c_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117de305d8165cf485982fb60580ecbf02c_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:01,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/be9facac0b7141a7a708b57c9b2b3f5e, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:01,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/be9facac0b7141a7a708b57c9b2b3f5e is 175, key is test_row_0/A:col10/1731813361387/Put/seqid=0 2024-11-17T03:16:01,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813361409, max timestamp=1731813361422Num uniques 859; 2024-11-17T03:16:01,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:16:01,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813361409, max timestamp=1731813361422Num uniques 859; 2024-11-17T03:16:01,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:16:01,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813361409, max timestamp=1731813361422Num uniques 859; 2024-11-17T03:16:01,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:16:01,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742129_1305 (size=39549) 2024-11-17T03:16:01,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813421448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813421450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813421452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813421453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813421454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:01,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:01,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813421556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813421556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813421566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813421577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813421577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:01,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:01,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813421766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813421766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813421787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813421787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813421787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:01,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/be9facac0b7141a7a708b57c9b2b3f5e 2024-11-17T03:16:01,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/345d9496f0fd48b7a887a4601970fc5e is 50, key is test_row_0/B:col10/1731813361387/Put/seqid=0 2024-11-17T03:16:01,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742130_1306 (size=12001) 2024-11-17T03:16:01,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:01,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813422076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813422076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813422097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813422107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813422107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/345d9496f0fd48b7a887a4601970fc5e 2024-11-17T03:16:02,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/e875be0169b24b4099b3075785c0aaa5 is 50, key is test_row_0/C:col10/1731813361387/Put/seqid=0 2024-11-17T03:16:02,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742131_1307 (size=12001) 2024-11-17T03:16:02,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:02,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:02,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813422597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813422597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813422607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813422627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813422627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:02,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/e875be0169b24b4099b3075785c0aaa5 2024-11-17T03:16:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/be9facac0b7141a7a708b57c9b2b3f5e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e 2024-11-17T03:16:02,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e, entries=200, sequenceid=15, filesize=38.6 K 2024-11-17T03:16:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/345d9496f0fd48b7a887a4601970fc5e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/345d9496f0fd48b7a887a4601970fc5e 2024-11-17T03:16:02,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/345d9496f0fd48b7a887a4601970fc5e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:16:02,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/e875be0169b24b4099b3075785c0aaa5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/e875be0169b24b4099b3075785c0aaa5 2024-11-17T03:16:02,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/e875be0169b24b4099b3075785c0aaa5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:16:02,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for aa45336f99df582886fbc013393f2fd9 in 1312ms, sequenceid=15, compaction requested=false 2024-11-17T03:16:02,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:02,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:02,758 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175a0c22e22fe74ef08af7967e3c927551_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813361451/Put/seqid=0 2024-11-17T03:16:02,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742132_1308 (size=12154) 2024-11-17T03:16:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:03,189 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175a0c22e22fe74ef08af7967e3c927551_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175a0c22e22fe74ef08af7967e3c927551_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:03,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/c5982a5b84dc4573b1b750ccf52a0d04, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:03,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/c5982a5b84dc4573b1b750ccf52a0d04 is 175, key is test_row_0/A:col10/1731813361451/Put/seqid=0 2024-11-17T03:16:03,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742133_1309 (size=30955) 2024-11-17T03:16:03,208 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/c5982a5b84dc4573b1b750ccf52a0d04 2024-11-17T03:16:03,210 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T03:16:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/9f6ae07c850049318adc641c6aa566d7 is 50, key is test_row_0/B:col10/1731813361451/Put/seqid=0 2024-11-17T03:16:03,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742134_1310 (size=12001) 2024-11-17T03:16:03,243 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/9f6ae07c850049318adc641c6aa566d7 2024-11-17T03:16:03,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/51a7bbedf82240d9a435ef416b8419e1 is 50, key is test_row_0/C:col10/1731813361451/Put/seqid=0 2024-11-17T03:16:03,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742135_1311 (size=12001) 2024-11-17T03:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:03,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:03,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813423639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813423646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813423647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813423647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,662 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813423651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,662 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,672 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/51a7bbedf82240d9a435ef416b8419e1 2024-11-17T03:16:03,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/c5982a5b84dc4573b1b750ccf52a0d04 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04 2024-11-17T03:16:03,683 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04, entries=150, sequenceid=51, filesize=30.2 K 2024-11-17T03:16:03,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/9f6ae07c850049318adc641c6aa566d7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9f6ae07c850049318adc641c6aa566d7 2024-11-17T03:16:03,688 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9f6ae07c850049318adc641c6aa566d7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T03:16:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/51a7bbedf82240d9a435ef416b8419e1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/51a7bbedf82240d9a435ef416b8419e1 2024-11-17T03:16:03,694 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/51a7bbedf82240d9a435ef416b8419e1, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T03:16:03,695 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for aa45336f99df582886fbc013393f2fd9 in 937ms, sequenceid=51, compaction requested=false 2024-11-17T03:16:03,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:03,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:03,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-17T03:16:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-17T03:16:03,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-17T03:16:03,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-17T03:16:03,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3240 sec 2024-11-17T03:16:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:03,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:16:03,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:03,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:03,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:03,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:03,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:03,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:03,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d1cc1ddab0c249568fe9f8787ad4884b_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:03,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742136_1312 (size=14594) 2024-11-17T03:16:03,778 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:03,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d1cc1ddab0c249568fe9f8787ad4884b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d1cc1ddab0c249568fe9f8787ad4884b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:03,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/0ad7b165d4a64d96914979d33acf1615, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:03,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/0ad7b165d4a64d96914979d33acf1615 is 175, key is test_row_0/A:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:03,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742137_1313 (size=39549) 2024-11-17T03:16:03,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813423814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813423815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813423819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813423936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813423937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813423937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813424146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813424146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813424157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/0ad7b165d4a64d96914979d33acf1615 2024-11-17T03:16:04,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/88a8dc304a8741268788d3e2a483411b is 50, key is test_row_0/B:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:04,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742138_1314 (size=12001) 2024-11-17T03:16:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813424457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813424457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813424467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:04,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/88a8dc304a8741268788d3e2a483411b 2024-11-17T03:16:04,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/2945608ca2664d448e0534cdfd83ce84 is 50, key is test_row_0/C:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:04,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742139_1315 (size=12001) 2024-11-17T03:16:04,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/2945608ca2664d448e0534cdfd83ce84 2024-11-17T03:16:04,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/0ad7b165d4a64d96914979d33acf1615 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615 2024-11-17T03:16:04,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615, entries=200, sequenceid=63, filesize=38.6 K 2024-11-17T03:16:04,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/88a8dc304a8741268788d3e2a483411b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/88a8dc304a8741268788d3e2a483411b 2024-11-17T03:16:04,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/88a8dc304a8741268788d3e2a483411b, entries=150, sequenceid=63, filesize=11.7 K 2024-11-17T03:16:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/2945608ca2664d448e0534cdfd83ce84 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2945608ca2664d448e0534cdfd83ce84 2024-11-17T03:16:04,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2945608ca2664d448e0534cdfd83ce84, entries=150, sequenceid=63, filesize=11.7 K 2024-11-17T03:16:04,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aa45336f99df582886fbc013393f2fd9 in 916ms, sequenceid=63, compaction requested=true 2024-11-17T03:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:04,677 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:04,678 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:04,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:04,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:04,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:04,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:04,682 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:04,682 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:04,682 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:04,682 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:04,682 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:04,682 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:04,682 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/345d9496f0fd48b7a887a4601970fc5e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9f6ae07c850049318adc641c6aa566d7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/88a8dc304a8741268788d3e2a483411b] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=35.2 K 2024-11-17T03:16:04,682 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=107.5 K 2024-11-17T03:16:04,682 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:04,682 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615] 2024-11-17T03:16:04,683 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9facac0b7141a7a708b57c9b2b3f5e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813361383 2024-11-17T03:16:04,683 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 345d9496f0fd48b7a887a4601970fc5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813361383 2024-11-17T03:16:04,683 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6ae07c850049318adc641c6aa566d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731813361438 2024-11-17T03:16:04,683 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5982a5b84dc4573b1b750ccf52a0d04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731813361438 2024-11-17T03:16:04,683 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a8dc304a8741268788d3e2a483411b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731813363638 2024-11-17T03:16:04,683 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad7b165d4a64d96914979d33acf1615, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731813363638 2024-11-17T03:16:04,694 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:04,695 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/39994be440604a9f9d0d818477d5adbc is 50, key is test_row_0/B:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:04,701 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:04,716 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117cc1406dd72e543f6b4172b6b5f45aaf0_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:04,718 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117cc1406dd72e543f6b4172b6b5f45aaf0_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:04,719 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cc1406dd72e543f6b4172b6b5f45aaf0_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:04,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742140_1316 (size=12104) 2024-11-17T03:16:04,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742141_1317 (size=4469) 2024-11-17T03:16:04,734 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/39994be440604a9f9d0d818477d5adbc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/39994be440604a9f9d0d818477d5adbc 2024-11-17T03:16:04,744 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into 39994be440604a9f9d0d818477d5adbc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:04,744 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:04,744 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=13, startTime=1731813364677; duration=0sec 2024-11-17T03:16:04,744 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:04,745 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:04,745 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:04,746 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:04,746 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:04,746 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:04,746 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/e875be0169b24b4099b3075785c0aaa5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/51a7bbedf82240d9a435ef416b8419e1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2945608ca2664d448e0534cdfd83ce84] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=35.2 K 2024-11-17T03:16:04,746 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e875be0169b24b4099b3075785c0aaa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813361383 2024-11-17T03:16:04,746 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a7bbedf82240d9a435ef416b8419e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731813361438 2024-11-17T03:16:04,747 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2945608ca2664d448e0534cdfd83ce84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731813363638 2024-11-17T03:16:04,757 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:04,758 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/1c93c3f1a7694d63862e3c435bba4e42 is 50, key is test_row_0/C:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:04,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742142_1318 (size=12104) 2024-11-17T03:16:04,775 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/1c93c3f1a7694d63862e3c435bba4e42 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/1c93c3f1a7694d63862e3c435bba4e42 2024-11-17T03:16:04,785 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into 1c93c3f1a7694d63862e3c435bba4e42(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:04,785 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:04,785 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=13, startTime=1731813364678; duration=0sec 2024-11-17T03:16:04,785 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:04,785 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:04,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T03:16:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:04,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:04,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:04,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ecff41c844104cd691f3b604dd010657_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813364979/Put/seqid=0 2024-11-17T03:16:04,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742143_1319 (size=14594) 2024-11-17T03:16:04,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:04,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ecff41c844104cd691f3b604dd010657_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ecff41c844104cd691f3b604dd010657_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:04,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2404b7abd2004bcc88c88cf9bbdcd97e, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:04,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2404b7abd2004bcc88c88cf9bbdcd97e is 175, key is test_row_0/A:col10/1731813364979/Put/seqid=0 2024-11-17T03:16:05,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742144_1320 (size=39549) 2024-11-17T03:16:05,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813424997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813425006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813425007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813425117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813425117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813425117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,130 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#271 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:05,130 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/57959104ceec4173a70374d1a5738d0c is 175, key is test_row_0/A:col10/1731813363638/Put/seqid=0 2024-11-17T03:16:05,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742145_1321 (size=31058) 2024-11-17T03:16:05,149 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/57959104ceec4173a70374d1a5738d0c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57959104ceec4173a70374d1a5738d0c 2024-11-17T03:16:05,154 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into 57959104ceec4173a70374d1a5738d0c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:05,154 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:05,154 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=13, startTime=1731813364677; duration=0sec 2024-11-17T03:16:05,154 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:05,154 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813425327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813425327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813425327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,403 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2404b7abd2004bcc88c88cf9bbdcd97e 2024-11-17T03:16:05,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/4f14dc35044a477eb9400f9825b0007a is 50, key is test_row_0/B:col10/1731813364979/Put/seqid=0 2024-11-17T03:16:05,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742146_1322 (size=12001) 2024-11-17T03:16:05,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/4f14dc35044a477eb9400f9825b0007a 2024-11-17T03:16:05,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/0a5ff04efa50496b94b0f9686add047e is 50, key is test_row_0/C:col10/1731813364979/Put/seqid=0 2024-11-17T03:16:05,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742147_1323 (size=12001) 2024-11-17T03:16:05,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/0a5ff04efa50496b94b0f9686add047e 2024-11-17T03:16:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2404b7abd2004bcc88c88cf9bbdcd97e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e 2024-11-17T03:16:05,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e, entries=200, sequenceid=86, filesize=38.6 K 2024-11-17T03:16:05,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/4f14dc35044a477eb9400f9825b0007a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4f14dc35044a477eb9400f9825b0007a 2024-11-17T03:16:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4f14dc35044a477eb9400f9825b0007a, entries=150, sequenceid=86, filesize=11.7 K 2024-11-17T03:16:05,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/0a5ff04efa50496b94b0f9686add047e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/0a5ff04efa50496b94b0f9686add047e 2024-11-17T03:16:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/0a5ff04efa50496b94b0f9686add047e, entries=150, sequenceid=86, filesize=11.7 K 2024-11-17T03:16:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for aa45336f99df582886fbc013393f2fd9 in 538ms, sequenceid=86, compaction requested=false 2024-11-17T03:16:05,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T03:16:05,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-17T03:16:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,540 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T03:16:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,541 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:05,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T03:16:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-17T03:16:05,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:05,694 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:16:05,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fc1476fce8e64888b93e6b458f716b2f_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_1/A:col10/1731813364996/Put/seqid=0 2024-11-17T03:16:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742148_1324 (size=9714) 2024-11-17T03:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:05,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813365771, max timestamp=1731813365794Num uniques 859; 2024-11-17T03:16:05,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:05,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813365771, max timestamp=1731813365794Num uniques 859; 2024-11-17T03:16:05,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:05,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813365771, max timestamp=1731813365794Num uniques 859; 2024-11-17T03:16:05,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813425821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813425822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813425822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813425830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813425831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T03:16:05,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813425937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813425947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813425947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813425947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813425957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:06,154 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117fc1476fce8e64888b93e6b458f716b2f_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fc1476fce8e64888b93e6b458f716b2f_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:06,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e643caf298d84a17b6a9ded307939958, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:06,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e643caf298d84a17b6a9ded307939958 is 175, key is test_row_1/A:col10/1731813364996/Put/seqid=0 2024-11-17T03:16:06,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742149_1325 (size=22361) 2024-11-17T03:16:06,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813426147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T03:16:06,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813426167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813426167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813426167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813426177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813426467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813426487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813426487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813426487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813426497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,559 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e643caf298d84a17b6a9ded307939958 2024-11-17T03:16:06,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/a64a3768e38343f080eb7f8379b3ba45 is 50, key is test_row_1/B:col10/1731813364996/Put/seqid=0 2024-11-17T03:16:06,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742150_1326 (size=9657) 2024-11-17T03:16:06,576 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/a64a3768e38343f080eb7f8379b3ba45 2024-11-17T03:16:06,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/69d51ca3e8ac461ea1a6a96114ca5e1b is 50, key is test_row_1/C:col10/1731813364996/Put/seqid=0 2024-11-17T03:16:06,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742151_1327 (size=9657) 2024-11-17T03:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T03:16:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813426987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:06,990 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/69d51ca3e8ac461ea1a6a96114ca5e1b 2024-11-17T03:16:06,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e643caf298d84a17b6a9ded307939958 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958 2024-11-17T03:16:06,999 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958, entries=100, sequenceid=98, filesize=21.8 K 2024-11-17T03:16:07,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/a64a3768e38343f080eb7f8379b3ba45 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a64a3768e38343f080eb7f8379b3ba45 2024-11-17T03:16:07,006 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a64a3768e38343f080eb7f8379b3ba45, entries=100, sequenceid=98, filesize=9.4 K 2024-11-17T03:16:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/69d51ca3e8ac461ea1a6a96114ca5e1b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/69d51ca3e8ac461ea1a6a96114ca5e1b 2024-11-17T03:16:07,011 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/69d51ca3e8ac461ea1a6a96114ca5e1b, entries=100, sequenceid=98, filesize=9.4 K 2024-11-17T03:16:07,012 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for aa45336f99df582886fbc013393f2fd9 in 1318ms, sequenceid=98, compaction requested=true 2024-11-17T03:16:07,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:07,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:07,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-17T03:16:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-17T03:16:07,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-17T03:16:07,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-17T03:16:07,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4780 sec 2024-11-17T03:16:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:07,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-17T03:16:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:07,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:07,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111732b9dab8fbb046d5aeea7750b8eea0b9_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:07,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742152_1328 (size=14744) 2024-11-17T03:16:07,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813427043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813427045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813427045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813427064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813427167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813427167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813427167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813427177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813427386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813427386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813427387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813427387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:07,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111732b9dab8fbb046d5aeea7750b8eea0b9_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111732b9dab8fbb046d5aeea7750b8eea0b9_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:07,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f56e39ea563e400588c56be5fb8301f6, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:07,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f56e39ea563e400588c56be5fb8301f6 is 175, key is test_row_0/A:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:07,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742153_1329 (size=39699) 2024-11-17T03:16:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T03:16:07,697 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:07,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-17T03:16:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T03:16:07,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:07,700 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:07,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:07,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813427707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813427708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813427708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813427709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T03:16:07,846 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f56e39ea563e400588c56be5fb8301f6 2024-11-17T03:16:07,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T03:16:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:07,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/1c5d937d947c425c9175aa4de878cd3b is 50, key is test_row_0/B:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:07,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:07,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742154_1330 (size=12101) 2024-11-17T03:16:07,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/1c5d937d947c425c9175aa4de878cd3b 2024-11-17T03:16:07,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/f10d103ee1314a388ea7828344344208 is 50, key is test_row_0/C:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:07,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742155_1331 (size=12101) 2024-11-17T03:16:08,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T03:16:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:08,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813427997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:08,013 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T03:16:08,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T03:16:08,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:08,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:08,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:08,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813428227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:08,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813428227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813428227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:08,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813428237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:08,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/f10d103ee1314a388ea7828344344208 2024-11-17T03:16:08,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f56e39ea563e400588c56be5fb8301f6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6 2024-11-17T03:16:08,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6, entries=200, sequenceid=133, filesize=38.8 K 2024-11-17T03:16:08,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/1c5d937d947c425c9175aa4de878cd3b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1c5d937d947c425c9175aa4de878cd3b 2024-11-17T03:16:08,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1c5d937d947c425c9175aa4de878cd3b, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T03:16:08,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/f10d103ee1314a388ea7828344344208 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/f10d103ee1314a388ea7828344344208 2024-11-17T03:16:08,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/f10d103ee1314a388ea7828344344208, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T03:16:08,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for aa45336f99df582886fbc013393f2fd9 in 1269ms, sequenceid=133, compaction requested=true 2024-11-17T03:16:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:08,290 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:08,290 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:08,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:08,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:08,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:08,292 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132667 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45863 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:08,293 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,293 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,293 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/39994be440604a9f9d0d818477d5adbc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4f14dc35044a477eb9400f9825b0007a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a64a3768e38343f080eb7f8379b3ba45, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1c5d937d947c425c9175aa4de878cd3b] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=44.8 K 2024-11-17T03:16:08,293 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57959104ceec4173a70374d1a5738d0c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=129.6 K 2024-11-17T03:16:08,293 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57959104ceec4173a70374d1a5738d0c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6] 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 39994be440604a9f9d0d818477d5adbc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731813363638 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57959104ceec4173a70374d1a5738d0c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731813363638 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f14dc35044a477eb9400f9825b0007a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731813363814 2024-11-17T03:16:08,293 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2404b7abd2004bcc88c88cf9bbdcd97e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731813363806 2024-11-17T03:16:08,294 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting a64a3768e38343f080eb7f8379b3ba45, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731813364996 2024-11-17T03:16:08,294 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e643caf298d84a17b6a9ded307939958, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731813364996 2024-11-17T03:16:08,294 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c5d937d947c425c9175aa4de878cd3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731813365819 2024-11-17T03:16:08,294 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56e39ea563e400588c56be5fb8301f6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731813365819 2024-11-17T03:16:08,301 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:08,302 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174d260be3e400483284bfbe2ba171e7cc_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:08,304 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:08,304 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/dcb463fe3b114abcb2cd13c04e0d96ea is 50, key is test_row_0/B:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:08,305 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174d260be3e400483284bfbe2ba171e7cc_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:08,305 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174d260be3e400483284bfbe2ba171e7cc_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:08,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T03:16:08,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,311 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:16:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:08,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742157_1333 (size=4469) 2024-11-17T03:16:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742156_1332 (size=12341) 2024-11-17T03:16:08,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cbe58d804b514c4180c60a38bf74de7b_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813367042/Put/seqid=0 2024-11-17T03:16:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T03:16:08,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742158_1334 (size=12304) 2024-11-17T03:16:08,718 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#282 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:08,719 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/0b36f771421f4557a1fd9ab5ee8af3dd is 175, key is test_row_0/A:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:08,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742159_1335 (size=31295) 2024-11-17T03:16:08,722 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/dcb463fe3b114abcb2cd13c04e0d96ea as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/dcb463fe3b114abcb2cd13c04e0d96ea 2024-11-17T03:16:08,727 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into dcb463fe3b114abcb2cd13c04e0d96ea(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:08,727 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:08,727 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=12, startTime=1731813368290; duration=0sec 2024-11-17T03:16:08,727 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:08,727 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:08,727 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:08,728 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45863 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:08,728 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:08,728 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,728 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/1c93c3f1a7694d63862e3c435bba4e42, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/0a5ff04efa50496b94b0f9686add047e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/69d51ca3e8ac461ea1a6a96114ca5e1b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/f10d103ee1314a388ea7828344344208] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=44.8 K 2024-11-17T03:16:08,728 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c93c3f1a7694d63862e3c435bba4e42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731813363638 2024-11-17T03:16:08,729 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a5ff04efa50496b94b0f9686add047e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731813363814 2024-11-17T03:16:08,729 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 69d51ca3e8ac461ea1a6a96114ca5e1b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731813364996 2024-11-17T03:16:08,729 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting f10d103ee1314a388ea7828344344208, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731813365819 2024-11-17T03:16:08,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,737 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cbe58d804b514c4180c60a38bf74de7b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cbe58d804b514c4180c60a38bf74de7b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2d39201cb2944f08a7b0a581dd067d0a, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2d39201cb2944f08a7b0a581dd067d0a is 175, key is test_row_0/A:col10/1731813367042/Put/seqid=0 2024-11-17T03:16:08,743 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:08,743 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/80eb0ccd6e994275ba5d72f9dfdddc98 is 50, key is test_row_0/C:col10/1731813365819/Put/seqid=0 2024-11-17T03:16:08,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742160_1336 (size=31105) 2024-11-17T03:16:08,752 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2d39201cb2944f08a7b0a581dd067d0a 2024-11-17T03:16:08,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/7942d7a83e584d248d0ea0b7e9610094 is 50, key is test_row_0/B:col10/1731813367042/Put/seqid=0 2024-11-17T03:16:08,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742161_1337 (size=12341) 2024-11-17T03:16:08,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742162_1338 (size=12151) 2024-11-17T03:16:08,774 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/7942d7a83e584d248d0ea0b7e9610094 2024-11-17T03:16:08,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/5d139d4f14b6497ebd3a7ef7a4f540c5 is 50, key is test_row_0/C:col10/1731813367042/Put/seqid=0 2024-11-17T03:16:08,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742163_1339 (size=12151) 2024-11-17T03:16:08,790 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/5d139d4f14b6497ebd3a7ef7a4f540c5 2024-11-17T03:16:08,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/2d39201cb2944f08a7b0a581dd067d0a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a 2024-11-17T03:16:08,799 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a, entries=150, sequenceid=143, filesize=30.4 K 2024-11-17T03:16:08,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/7942d7a83e584d248d0ea0b7e9610094 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/7942d7a83e584d248d0ea0b7e9610094 2024-11-17T03:16:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,802 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/7942d7a83e584d248d0ea0b7e9610094, entries=150, sequenceid=143, filesize=11.9 K 2024-11-17T03:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/5d139d4f14b6497ebd3a7ef7a4f540c5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/5d139d4f14b6497ebd3a7ef7a4f540c5 2024-11-17T03:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,807 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/5d139d4f14b6497ebd3a7ef7a4f540c5, entries=150, sequenceid=143, filesize=11.9 K 2024-11-17T03:16:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,807 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for aa45336f99df582886fbc013393f2fd9 in 496ms, sequenceid=143, compaction requested=false 2024-11-17T03:16:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:08,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T03:16:08,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1090 sec 2024-11-17T03:16:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.1140 sec 2024-11-17T03:16:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T03:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,837 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T03:16:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,840 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,840 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T03:16:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T03:16:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:08,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-17T03:16:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-17T03:16:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 159 msec 2024-11-17T03:16:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,129 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/0b36f771421f4557a1fd9ab5ee8af3dd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0b36f771421f4557a1fd9ab5ee8af3dd 2024-11-17T03:16:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,134 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into 0b36f771421f4557a1fd9ab5ee8af3dd(size=30.6 K), total size for store is 60.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,134 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:09,134 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=12, startTime=1731813368290; duration=0sec 2024-11-17T03:16:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,134 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:09,134 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T03:16:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,157 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-17T03:16:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T03:16:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,161 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,174 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/80eb0ccd6e994275ba5d72f9dfdddc98 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/80eb0ccd6e994275ba5d72f9dfdddc98 2024-11-17T03:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,178 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into 80eb0ccd6e994275ba5d72f9dfdddc98(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:09,178 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:09,178 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=12, startTime=1731813368291; duration=0sec 2024-11-17T03:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,179 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:09,179 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T03:16:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T03:16:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,315 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T03:16:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:09,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117737386070949459ca1461c6355a3710b_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813369303/Put/seqid=0 2024-11-17T03:16:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742164_1340 (size=14794) 2024-11-17T03:16:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:09,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813369404, max timestamp=1731813369426Num uniques 859; 2024-11-17T03:16:09,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813369404, max timestamp=1731813369426Num uniques 859; 2024-11-17T03:16:09,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813369404, max timestamp=1731813369426Num uniques 859; 2024-11-17T03:16:09,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:09,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:09,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:09,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813429464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731813369458, max timestamp=1731813369472Num uniques 1118; 2024-11-17T03:16:09,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-17T03:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813429464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731813369458, max timestamp=1731813369472Num uniques 1118; 2024-11-17T03:16:09,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-17T03:16:09,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731813369458, max timestamp=1731813369472Num uniques 1118; 2024-11-17T03:16:09,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-17T03:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T03:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813369525, max timestamp=1731813369540Num uniques 1077; 2024-11-17T03:16:09,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T03:16:09,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813369525, max timestamp=1731813369540Num uniques 1077; 2024-11-17T03:16:09,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T03:16:09,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731813369525, max timestamp=1731813369540Num uniques 1077; 2024-11-17T03:16:09,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T03:16:09,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731813369576, max timestamp=1731813369600Num uniques 1036; 2024-11-17T03:16:09,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-17T03:16:09,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731813369576, max timestamp=1731813369600Num uniques 1036; 2024-11-17T03:16:09,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-17T03:16:09,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731813369593, max timestamp=1731813369600Num uniques 1036; 2024-11-17T03:16:09,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-17T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731813369630, max timestamp=1731813369650Num uniques 1009; 2024-11-17T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=15800, suffix heap size=132904,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=345 2024-11-17T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731813369630, max timestamp=1731813369650Num uniques 1145; 2024-11-17T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-17T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731813369630, max timestamp=1731813369650Num uniques 1145; 2024-11-17T03:16:09,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=15800, suffix heap size=147048,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=345 2024-11-17T03:16:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813429702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813429702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813429703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813429706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:09,740 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117737386070949459ca1461c6355a3710b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117737386070949459ca1461c6355a3710b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/034e8cf737e14910b8fb1c983374c029, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/034e8cf737e14910b8fb1c983374c029 is 175, key is test_row_0/A:col10/1731813369303/Put/seqid=0 2024-11-17T03:16:09,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742165_1341 (size=39745) 2024-11-17T03:16:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T03:16:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813429817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813429817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813429817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813429818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813430027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813430027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813430027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813430027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813430036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,048 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,145 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/034e8cf737e14910b8fb1c983374c029 2024-11-17T03:16:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/536290bd047b4f1fb286d4239b1d4864 is 50, key is test_row_0/B:col10/1731813369303/Put/seqid=0 2024-11-17T03:16:10,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742166_1342 (size=9757) 2024-11-17T03:16:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T03:16:10,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,555 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/536290bd047b4f1fb286d4239b1d4864 2024-11-17T03:16:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/d7374d2976d44d89aad10e4fc2c6cf7e is 50, key is test_row_0/C:col10/1731813369303/Put/seqid=0 2024-11-17T03:16:10,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742167_1343 (size=9757) 2024-11-17T03:16:10,581 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/d7374d2976d44d89aad10e4fc2c6cf7e 2024-11-17T03:16:10,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/034e8cf737e14910b8fb1c983374c029 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029 2024-11-17T03:16:10,592 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029, entries=200, sequenceid=154, filesize=38.8 K 2024-11-17T03:16:10,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/536290bd047b4f1fb286d4239b1d4864 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/536290bd047b4f1fb286d4239b1d4864 2024-11-17T03:16:10,598 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/536290bd047b4f1fb286d4239b1d4864, entries=100, sequenceid=154, filesize=9.5 K 2024-11-17T03:16:10,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/d7374d2976d44d89aad10e4fc2c6cf7e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d7374d2976d44d89aad10e4fc2c6cf7e 2024-11-17T03:16:10,620 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d7374d2976d44d89aad10e4fc2c6cf7e, entries=100, sequenceid=154, filesize=9.5 K 2024-11-17T03:16:10,621 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for aa45336f99df582886fbc013393f2fd9 in 1305ms, sequenceid=154, compaction requested=true 2024-11-17T03:16:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:10,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-17T03:16:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-17T03:16:10,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T03:16:10,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-17T03:16:10,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4680 sec 2024-11-17T03:16:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:10,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=167.72 KB heapSize=467.86 KB 2024-11-17T03:16:10,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:10,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:16:10,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:10,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:16:10,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:10,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T03:16:10,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172bbbaaadca534895b7d4603fcf7ffa41_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742168_1344 (size=14994) 2024-11-17T03:16:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813430879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813430879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813430880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813430881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813430997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813430997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813430997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813430997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813431207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813431207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813431207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813431217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:11,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172bbbaaadca534895b7d4603fcf7ffa41_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172bbbaaadca534895b7d4603fcf7ffa41_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:11,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/70494319ed0945558a84d7e0a7f4b8ba, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:11,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/70494319ed0945558a84d7e0a7f4b8ba is 175, key is test_row_0/A:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:11,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742169_1345 (size=39949) 2024-11-17T03:16:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T03:16:11,307 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:11,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T03:16:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:11,309 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:11,310 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:11,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:11,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813431527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813431527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813431527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813431537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:11,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:11,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:11,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:11,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:11,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/70494319ed0945558a84d7e0a7f4b8ba 2024-11-17T03:16:11,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/897730ae150d4e6fb99a981fe14f4226 is 50, key is test_row_0/B:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:11,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742170_1346 (size=12301) 2024-11-17T03:16:11,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:11,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813432037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813432037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813432037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:12,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813432057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:12,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:12,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/897730ae150d4e6fb99a981fe14f4226 2024-11-17T03:16:12,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/4d6b56cb26114f2195b78bfd9586de6c is 50, key is test_row_0/C:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:12,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742171_1347 (size=12301) 2024-11-17T03:16:12,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:12,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/4d6b56cb26114f2195b78bfd9586de6c 2024-11-17T03:16:12,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/70494319ed0945558a84d7e0a7f4b8ba as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba 2024-11-17T03:16:12,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba, entries=200, sequenceid=261, filesize=39.0 K 2024-11-17T03:16:12,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/897730ae150d4e6fb99a981fe14f4226 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/897730ae150d4e6fb99a981fe14f4226 2024-11-17T03:16:12,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:12,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/897730ae150d4e6fb99a981fe14f4226, entries=150, sequenceid=261, filesize=12.0 K 2024-11-17T03:16:12,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/4d6b56cb26114f2195b78bfd9586de6c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4d6b56cb26114f2195b78bfd9586de6c 2024-11-17T03:16:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4d6b56cb26114f2195b78bfd9586de6c, entries=150, sequenceid=261, filesize=12.0 K 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~467.81 KB/479040, currentSize=20.13 KB/20610 for aa45336f99df582886fbc013393f2fd9 in 1662ms, sequenceid=261, compaction requested=true 2024-11-17T03:16:12,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,538 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:12,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:12,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:12,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:12,539 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:12,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:12,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,539 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142094 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:12,539 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,539 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:12,539 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,539 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,539 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,540 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/dcb463fe3b114abcb2cd13c04e0d96ea, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/7942d7a83e584d248d0ea0b7e9610094, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/536290bd047b4f1fb286d4239b1d4864, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/897730ae150d4e6fb99a981fe14f4226] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=45.5 K 2024-11-17T03:16:12,540 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0b36f771421f4557a1fd9ab5ee8af3dd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=138.8 K 2024-11-17T03:16:12,540 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,540 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0b36f771421f4557a1fd9ab5ee8af3dd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba] 2024-11-17T03:16:12,540 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb463fe3b114abcb2cd13c04e0d96ea, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731813365819 2024-11-17T03:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,540 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b36f771421f4557a1fd9ab5ee8af3dd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731813365819 2024-11-17T03:16:12,540 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7942d7a83e584d248d0ea0b7e9610094, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731813367042 2024-11-17T03:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,540 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 536290bd047b4f1fb286d4239b1d4864, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731813369303 2024-11-17T03:16:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,541 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 897730ae150d4e6fb99a981fe14f4226, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731813369701 2024-11-17T03:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,541 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d39201cb2944f08a7b0a581dd067d0a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731813367042 2024-11-17T03:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,542 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 034e8cf737e14910b8fb1c983374c029, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731813369299 2024-11-17T03:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,542 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70494319ed0945558a84d7e0a7f4b8ba, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731813369701 2024-11-17T03:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,550 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,550 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,550 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/4fe083bde06246fbb2a3ae7cf856d6f4 is 50, key is test_row_0/B:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,552 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117970658b12c6b43179a4c2cb823ffd350_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,554 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117970658b12c6b43179a4c2cb823ffd350_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,555 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117970658b12c6b43179a4c2cb823ffd350_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742172_1348 (size=12677) 2024-11-17T03:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,577 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/4fe083bde06246fbb2a3ae7cf856d6f4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4fe083bde06246fbb2a3ae7cf856d6f4 2024-11-17T03:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742173_1349 (size=4469) 2024-11-17T03:16:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,580 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#295 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,580 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/1e1511e1f2f34621aa3199423613271a is 175, key is test_row_0/A:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,584 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into 4fe083bde06246fbb2a3ae7cf856d6f4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:12,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:12,584 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=12, startTime=1731813372538; duration=0sec 2024-11-17T03:16:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:12,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:12,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,585 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,585 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:12,585 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,585 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/80eb0ccd6e994275ba5d72f9dfdddc98, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/5d139d4f14b6497ebd3a7ef7a4f540c5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d7374d2976d44d89aad10e4fc2c6cf7e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4d6b56cb26114f2195b78bfd9586de6c] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=45.5 K 2024-11-17T03:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,586 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 80eb0ccd6e994275ba5d72f9dfdddc98, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731813365819 2024-11-17T03:16:12,586 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d139d4f14b6497ebd3a7ef7a4f540c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731813367042 2024-11-17T03:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,586 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting d7374d2976d44d89aad10e4fc2c6cf7e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731813369303 2024-11-17T03:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,587 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d6b56cb26114f2195b78bfd9586de6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731813369701 2024-11-17T03:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,610 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,611 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/b9ee84e567ba454ea028c92b73f38915 is 50, key is test_row_0/C:col10/1731813369701/Put/seqid=0 2024-11-17T03:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742174_1350 (size=31631) 2024-11-17T03:16:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742175_1351 (size=12677) 2024-11-17T03:16:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,646 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/b9ee84e567ba454ea028c92b73f38915 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/b9ee84e567ba454ea028c92b73f38915 2024-11-17T03:16:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,652 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into b9ee84e567ba454ea028c92b73f38915(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:12,652 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:12,652 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=12, startTime=1731813372539; duration=0sec 2024-11-17T03:16:12,652 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:12,652 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,684 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-17T03:16:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:12,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117423336cc59624960bdded85f6820671b_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813370880/Put/seqid=0 2024-11-17T03:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742176_1352 (size=9914) 2024-11-17T03:16:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,707 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117423336cc59624960bdded85f6820671b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117423336cc59624960bdded85f6820671b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/57e6082e93804aee8ef28d07749aaf43, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/57e6082e93804aee8ef28d07749aaf43 is 175, key is test_row_0/A:col10/1731813370880/Put/seqid=0 2024-11-17T03:16:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742177_1353 (size=22561) 2024-11-17T03:16:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,037 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/1e1511e1f2f34621aa3199423613271a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1e1511e1f2f34621aa3199423613271a 2024-11-17T03:16:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,043 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into 1e1511e1f2f34621aa3199423613271a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:13,043 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:13,043 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=12, startTime=1731813372538; duration=0sec 2024-11-17T03:16:13,043 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:13,043 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:13,123 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/57e6082e93804aee8ef28d07749aaf43 2024-11-17T03:16:13,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/5292f694648847f3a487ad9e56972e9a is 50, key is test_row_0/B:col10/1731813370880/Put/seqid=0 2024-11-17T03:16:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813373141, max timestamp=1731813373165Num uniques 859; 2024-11-17T03:16:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731813373141, max timestamp=1731813373165Num uniques 859; 2024-11-17T03:16:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:13,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:13,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731813373141, max timestamp=1731813373165Num uniques 859; 2024-11-17T03:16:13,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T03:16:13,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742178_1354 (size=9857) 2024-11-17T03:16:13,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731813373201, max timestamp=1731813373236Num uniques 1118; 2024-11-17T03:16:13,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-17T03:16:13,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731813373201, max timestamp=1731813373236Num uniques 1118; 2024-11-17T03:16:13,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-17T03:16:13,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731813373201, max timestamp=1731813373236Num uniques 1118; 2024-11-17T03:16:13,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-17T03:16:13,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731813373261, max timestamp=1731813373298Num uniques 1077; 2024-11-17T03:16:13,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-17T03:16:13,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731813373261, max timestamp=1731813373298Num uniques 1077; 2024-11-17T03:16:13,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-17T03:16:13,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731813373261, max timestamp=1731813373298Num uniques 1077; 2024-11-17T03:16:13,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-17T03:16:13,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731813373328, max timestamp=1731813373350Num uniques 1136; 2024-11-17T03:16:13,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731813373328, max timestamp=1731813373350Num uniques 1136; 2024-11-17T03:16:13,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-17T03:16:13,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-17T03:16:13,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731813373328, max timestamp=1731813373350Num uniques 1136; 2024-11-17T03:16:13,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=15386, suffix heap size=146112,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=336 2024-11-17T03:16:13,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731813373391, max timestamp=1731813373409Num uniques 1195; 2024-11-17T03:16:13,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731813373391, max timestamp=1731813373409Num uniques 1195; 2024-11-17T03:16:13,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731813373391, max timestamp=1731813373409Num uniques 1195; 2024-11-17T03:16:13,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-17T03:16:13,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-17T03:16:13,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=13510, suffix heap size=152248,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=295 2024-11-17T03:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813433459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813433461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813433468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:13,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731813373450, max timestamp=1731813373476Num uniques 1154; 2024-11-17T03:16:13,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731813373450, max timestamp=1731813373476Num uniques 1154; 2024-11-17T03:16:13,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731813373450, max timestamp=1731813373476Num uniques 1154; 2024-11-17T03:16:13,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-17T03:16:13,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-17T03:16:13,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-17T03:16:13,577 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/5292f694648847f3a487ad9e56972e9a 2024-11-17T03:16:13,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/da49435a96304600a17c256359d69d78 is 50, key is test_row_0/C:col10/1731813370880/Put/seqid=0 2024-11-17T03:16:13,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742179_1355 (size=9857) 2024-11-17T03:16:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731813373595, max timestamp=1731813373615Num uniques 1013; 2024-11-17T03:16:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T03:16:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731813373595, max timestamp=1731813373615Num uniques 1013; 2024-11-17T03:16:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T03:16:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731813373597, max timestamp=1731813373615Num uniques 1013; 2024-11-17T03:16:13,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-17T03:16:13,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731813373641, max timestamp=1731813373655Num uniques 1022; 2024-11-17T03:16:13,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=14742, suffix heap size=134256,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=322 2024-11-17T03:16:13,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731813373641, max timestamp=1731813373655Num uniques 1072; 2024-11-17T03:16:13,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=14742, suffix heap size=139456,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=322 2024-11-17T03:16:13,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=322, cellSize=14742, totalHeapSize=33728, min timestamp=1731813373641, max timestamp=1731813373655Num uniques 1072; 2024-11-17T03:16:13,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=14742, suffix heap size=139456,new segment heap size=33728 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=322 2024-11-17T03:16:13,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731813373678, max timestamp=1731813373706Num uniques 1181; 2024-11-17T03:16:13,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731813373678, max timestamp=1731813373706Num uniques 1181; 2024-11-17T03:16:13,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=12866, suffix heap size=150792,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=281 2024-11-17T03:16:13,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=12866, suffix heap size=150792,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=281 2024-11-17T03:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731813373678, max timestamp=1731813373706Num uniques 1181; 2024-11-17T03:16:13,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54086, new segment data size=12866, suffix heap size=150792,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1181, new segment cells count=281 2024-11-17T03:16:13,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731813373728, max timestamp=1731813373764Num uniques 1140; 2024-11-17T03:16:13,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-17T03:16:13,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731813373728, max timestamp=1731813373764Num uniques 1140; 2024-11-17T03:16:13,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-17T03:16:13,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731813373728, max timestamp=1731813373764Num uniques 1140; 2024-11-17T03:16:13,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-17T03:16:13,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731813373728, max timestamp=1731813373821Num uniques 1099; 2024-11-17T03:16:13,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-17T03:16:13,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731813373728, max timestamp=1731813373821Num uniques 1099; 2024-11-17T03:16:13,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-17T03:16:13,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731813373728, max timestamp=1731813373821Num uniques 1099; 2024-11-17T03:16:13,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=13694, suffix heap size=142264,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=299 2024-11-17T03:16:13,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:13,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813433864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813433865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:13,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731813373858, max timestamp=1731813373874Num uniques 1158; 2024-11-17T03:16:13,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731813373858, max timestamp=1731813373874Num uniques 1158; 2024-11-17T03:16:13,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731813373858, max timestamp=1731813373874Num uniques 1158; 2024-11-17T03:16:13,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-11-17T03:16:13,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-11-17T03:16:13,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=9520, suffix heap size=148400,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=208 2024-11-17T03:16:13,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813373909, max timestamp=1731813373938Num uniques 1067; 2024-11-17T03:16:13,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813373909, max timestamp=1731813373938Num uniques 1067; 2024-11-17T03:16:13,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-17T03:16:13,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-17T03:16:13,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731813373909, max timestamp=1731813373938Num uniques 1067; 2024-11-17T03:16:13,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-11-17T03:16:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731813373976, max timestamp=1731813373996Num uniques 1076; 2024-11-17T03:16:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731813373976, max timestamp=1731813373996Num uniques 1076; 2024-11-17T03:16:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=14926, suffix heap size=139872,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=326 2024-11-17T03:16:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=14926, suffix heap size=139872,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=326 2024-11-17T03:16:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731813373976, max timestamp=1731813373996Num uniques 1076; 2024-11-17T03:16:14,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=14926, suffix heap size=139872,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=326 2024-11-17T03:16:14,001 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/da49435a96304600a17c256359d69d78 2024-11-17T03:16:14,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/57e6082e93804aee8ef28d07749aaf43 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43 2024-11-17T03:16:14,026 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43, entries=100, sequenceid=269, filesize=22.0 K 2024-11-17T03:16:14,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/5292f694648847f3a487ad9e56972e9a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/5292f694648847f3a487ad9e56972e9a 2024-11-17T03:16:14,031 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/5292f694648847f3a487ad9e56972e9a, entries=100, sequenceid=269, filesize=9.6 K 2024-11-17T03:16:14,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/da49435a96304600a17c256359d69d78 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/da49435a96304600a17c256359d69d78 2024-11-17T03:16:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/da49435a96304600a17c256359d69d78, entries=100, sequenceid=269, filesize=9.6 K 2024-11-17T03:16:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=87.22 KB/89310 for aa45336f99df582886fbc013393f2fd9 in 1354ms, sequenceid=269, compaction requested=false 2024-11-17T03:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T03:16:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T03:16:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:14,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=93.93 KB heapSize=249.80 KB 2024-11-17T03:16:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:14,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T03:16:14,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7300 sec 2024-11-17T03:16:14,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7350 sec 2024-11-17T03:16:14,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117aa634fab5d0e47b3b3e09b1f43692107_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:14,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742180_1356 (size=12454) 2024-11-17T03:16:14,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:14,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117aa634fab5d0e47b3b3e09b1f43692107_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aa634fab5d0e47b3b3e09b1f43692107_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:14,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/4b0b3603b7194c87bef2ea3cfe9a34e1, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:14,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/4b0b3603b7194c87bef2ea3cfe9a34e1 is 175, key is test_row_0/A:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:14,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742181_1357 (size=31255) 2024-11-17T03:16:14,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=521, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/4b0b3603b7194c87bef2ea3cfe9a34e1 2024-11-17T03:16:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813434107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813434110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813434118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813434119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/3d84798fc87d448ab6684bdb751fb32d is 50, key is test_row_0/B:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742182_1358 (size=12301) 2024-11-17T03:16:14,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813434120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813434227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813434227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813434227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813434227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813434237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813434447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813434447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813434447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813434447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813434456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/3d84798fc87d448ab6684bdb751fb32d 2024-11-17T03:16:14,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/d3b29b2a71414dfdb2df57cbd274b974 is 50, key is test_row_0/C:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:14,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742183_1359 (size=12301) 2024-11-17T03:16:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813434767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813434767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813434767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813434767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813434776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:14,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/d3b29b2a71414dfdb2df57cbd274b974 2024-11-17T03:16:14,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/4b0b3603b7194c87bef2ea3cfe9a34e1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1 2024-11-17T03:16:14,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1, entries=150, sequenceid=521, filesize=30.5 K 2024-11-17T03:16:14,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/3d84798fc87d448ab6684bdb751fb32d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/3d84798fc87d448ab6684bdb751fb32d 2024-11-17T03:16:14,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/3d84798fc87d448ab6684bdb751fb32d, entries=150, sequenceid=521, filesize=12.0 K 2024-11-17T03:16:14,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/d3b29b2a71414dfdb2df57cbd274b974 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d3b29b2a71414dfdb2df57cbd274b974 2024-11-17T03:16:14,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d3b29b2a71414dfdb2df57cbd274b974, entries=150, sequenceid=521, filesize=12.0 K 2024-11-17T03:16:14,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~249.75 KB/255744, currentSize=93.93 KB/96180 for aa45336f99df582886fbc013393f2fd9 in 934ms, sequenceid=521, compaction requested=true 2024-11-17T03:16:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:14,975 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:14,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:14,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:14,975 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:14,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:14,977 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:14,977 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:14,977 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:14,977 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1e1511e1f2f34621aa3199423613271a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=83.4 K 2024-11-17T03:16:14,977 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:14,977 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1e1511e1f2f34621aa3199423613271a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1] 2024-11-17T03:16:14,977 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:14,977 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:14,977 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:14,978 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4fe083bde06246fbb2a3ae7cf856d6f4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/5292f694648847f3a487ad9e56972e9a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/3d84798fc87d448ab6684bdb751fb32d] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=34.0 K 2024-11-17T03:16:14,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:14,978 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e1511e1f2f34621aa3199423613271a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731813369701 2024-11-17T03:16:14,979 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe083bde06246fbb2a3ae7cf856d6f4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731813369701 2024-11-17T03:16:14,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:14,979 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5292f694648847f3a487ad9e56972e9a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731813370877 2024-11-17T03:16:14,979 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e6082e93804aee8ef28d07749aaf43, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731813370877 2024-11-17T03:16:14,979 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d84798fc87d448ab6684bdb751fb32d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731813374021 2024-11-17T03:16:14,980 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b0b3603b7194c87bef2ea3cfe9a34e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731813374021 2024-11-17T03:16:14,993 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:15,001 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a5163707d44c4f128e775095ea2b0cd2_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:15,002 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:15,003 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a5163707d44c4f128e775095ea2b0cd2_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:15,003 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a5163707d44c4f128e775095ea2b0cd2_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:15,003 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/caaaafe2ff3e4a8bbb54fbc9e579ae73 is 50, key is test_row_0/B:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:15,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742184_1360 (size=12779) 2024-11-17T03:16:15,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742185_1361 (size=4469) 2024-11-17T03:16:15,035 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#303 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:15,035 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/736a2cc22db6419cb5c014cdae47a8d5 is 175, key is test_row_0/A:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:15,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742186_1362 (size=31733) 2024-11-17T03:16:15,063 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/736a2cc22db6419cb5c014cdae47a8d5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/736a2cc22db6419cb5c014cdae47a8d5 2024-11-17T03:16:15,069 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into 736a2cc22db6419cb5c014cdae47a8d5(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:15,069 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:15,069 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=13, startTime=1731813374975; duration=0sec 2024-11-17T03:16:15,070 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:15,070 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:15,070 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:15,070 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:15,071 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:15,071 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:15,071 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/b9ee84e567ba454ea028c92b73f38915, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/da49435a96304600a17c256359d69d78, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d3b29b2a71414dfdb2df57cbd274b974] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=34.0 K 2024-11-17T03:16:15,071 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ee84e567ba454ea028c92b73f38915, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731813369701 2024-11-17T03:16:15,071 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting da49435a96304600a17c256359d69d78, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731813370877 2024-11-17T03:16:15,071 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b29b2a71414dfdb2df57cbd274b974, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731813374021 2024-11-17T03:16:15,078 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:15,079 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/8a6eed3db29d4b8e84f6405170cd114c is 50, key is test_row_0/C:col10/1731813374021/Put/seqid=0 2024-11-17T03:16:15,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742187_1363 (size=12779) 2024-11-17T03:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:15,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:16:15,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:15,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:15,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:15,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175508fbc1bc1146f1baa540279cfe0158_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813374117/Put/seqid=0 2024-11-17T03:16:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742188_1364 (size=14994) 2024-11-17T03:16:15,303 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:15,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175508fbc1bc1146f1baa540279cfe0158_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175508fbc1bc1146f1baa540279cfe0158_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:15,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f6332baba009405e97c41f064b22b84b, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:15,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f6332baba009405e97c41f064b22b84b is 175, key is test_row_0/A:col10/1731813374117/Put/seqid=0 2024-11-17T03:16:15,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813435307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742189_1365 (size=39949) 2024-11-17T03:16:15,314 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=540, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f6332baba009405e97c41f064b22b84b 2024-11-17T03:16:15,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813435308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813435310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813435311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813435313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/9c93f9188c1140d9b8d48ece2e1be219 is 50, key is test_row_0/B:col10/1731813374117/Put/seqid=0 2024-11-17T03:16:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742190_1366 (size=12301) 2024-11-17T03:16:15,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813435417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813435427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813435427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,440 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/caaaafe2ff3e4a8bbb54fbc9e579ae73 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/caaaafe2ff3e4a8bbb54fbc9e579ae73 2024-11-17T03:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,446 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into caaaafe2ff3e4a8bbb54fbc9e579ae73(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:15,446 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:15,446 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=13, startTime=1731813374975; duration=0sec 2024-11-17T03:16:15,446 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:15,446 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813435437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813435437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T03:16:15,478 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:15,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T03:16:15,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T03:16:15,481 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:15,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:15,492 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/8a6eed3db29d4b8e84f6405170cd114c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8a6eed3db29d4b8e84f6405170cd114c 2024-11-17T03:16:15,498 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into 8a6eed3db29d4b8e84f6405170cd114c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:15,498 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:15,498 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=13, startTime=1731813374977; duration=0sec 2024-11-17T03:16:15,498 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:15,499 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T03:16:15,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T03:16:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813435627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:15,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813435647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813435647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813435657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813435657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:15,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/9c93f9188c1140d9b8d48ece2e1be219 2024-11-17T03:16:15,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/662b161e0aa0495b9ccdcb45c1eaf060 is 50, key is test_row_0/C:col10/1731813374117/Put/seqid=0 2024-11-17T03:16:15,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T03:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:15,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:15,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T03:16:15,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742191_1367 (size=12301) 2024-11-17T03:16:15,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/662b161e0aa0495b9ccdcb45c1eaf060 2024-11-17T03:16:15,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f6332baba009405e97c41f064b22b84b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b 2024-11-17T03:16:15,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b, entries=200, sequenceid=540, filesize=39.0 K 2024-11-17T03:16:15,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/9c93f9188c1140d9b8d48ece2e1be219 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9c93f9188c1140d9b8d48ece2e1be219 2024-11-17T03:16:15,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9c93f9188c1140d9b8d48ece2e1be219, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T03:16:15,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/662b161e0aa0495b9ccdcb45c1eaf060 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/662b161e0aa0495b9ccdcb45c1eaf060 2024-11-17T03:16:15,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/662b161e0aa0495b9ccdcb45c1eaf060, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T03:16:15,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aa45336f99df582886fbc013393f2fd9 in 557ms, sequenceid=540, compaction requested=false 2024-11-17T03:16:15,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:15,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T03:16:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:15,947 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:16:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:15,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:15,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b50fa86c3a14f03ad1d619c1fa6c181_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813375309/Put/seqid=0 2024-11-17T03:16:16,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742192_1368 (size=12454) 2024-11-17T03:16:16,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813436022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813436024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813436037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813436038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813436048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T03:16:16,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813436157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813436157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813436157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813436157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813436178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813436387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813436387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813436387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813436387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813436397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:16,414 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175b50fa86c3a14f03ad1d619c1fa6c181_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b50fa86c3a14f03ad1d619c1fa6c181_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:16,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/ecb2af504ba84e70a57fb4c8f50fb736, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:16,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/ecb2af504ba84e70a57fb4c8f50fb736 is 175, key is test_row_0/A:col10/1731813375309/Put/seqid=0 2024-11-17T03:16:16,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742193_1369 (size=31255) 2024-11-17T03:16:16,427 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=556, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/ecb2af504ba84e70a57fb4c8f50fb736 2024-11-17T03:16:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/90d3611df93a41a4bf626dbd3cbca5ef is 50, key is test_row_0/B:col10/1731813375309/Put/seqid=0 2024-11-17T03:16:16,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742194_1370 (size=12301) 2024-11-17T03:16:16,448 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/90d3611df93a41a4bf626dbd3cbca5ef 2024-11-17T03:16:16,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/42b1fab13f1d44c9acf0ee22ae299cc1 is 50, key is test_row_0/C:col10/1731813375309/Put/seqid=0 2024-11-17T03:16:16,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742195_1371 (size=12301) 2024-11-17T03:16:16,468 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/42b1fab13f1d44c9acf0ee22ae299cc1 2024-11-17T03:16:16,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/ecb2af504ba84e70a57fb4c8f50fb736 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736 2024-11-17T03:16:16,477 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736, entries=150, sequenceid=556, filesize=30.5 K 2024-11-17T03:16:16,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/90d3611df93a41a4bf626dbd3cbca5ef as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/90d3611df93a41a4bf626dbd3cbca5ef 2024-11-17T03:16:16,483 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/90d3611df93a41a4bf626dbd3cbca5ef, entries=150, sequenceid=556, filesize=12.0 K 2024-11-17T03:16:16,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/42b1fab13f1d44c9acf0ee22ae299cc1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/42b1fab13f1d44c9acf0ee22ae299cc1 2024-11-17T03:16:16,491 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/42b1fab13f1d44c9acf0ee22ae299cc1, entries=150, sequenceid=556, filesize=12.0 K 2024-11-17T03:16:16,492 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aa45336f99df582886fbc013393f2fd9 in 546ms, sequenceid=556, compaction requested=true 2024-11-17T03:16:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T03:16:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T03:16:16,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T03:16:16,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-17T03:16:16,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0200 sec 2024-11-17T03:16:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T03:16:16,617 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:16,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T03:16:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:16,619 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:16,620 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:16,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:16,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T03:16:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:16,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111790aff0ab9ced43b58bff8384d6c2316d_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813436745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813436747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813436758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:16,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:16,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813436764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813436765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742196_1372 (size=14994) 2024-11-17T03:16:16,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:16,807 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111790aff0ab9ced43b58bff8384d6c2316d_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111790aff0ab9ced43b58bff8384d6c2316d_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:16,808 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/b0d48327c58a4a6c8b0fdefcd590c324, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:16,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/b0d48327c58a4a6c8b0fdefcd590c324 is 175, key is test_row_0/A:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:16,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742197_1373 (size=39949) 2024-11-17T03:16:16,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813436870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813436879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813436887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813436887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813436887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:16,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:16,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:17,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:17,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813437087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813437097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813437117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813437117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813437117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:17,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=575, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/b0d48327c58a4a6c8b0fdefcd590c324 2024-11-17T03:16:17,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/cb30e77b753b4cada9f7753b39c6f394 is 50, key is test_row_0/B:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:17,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742198_1374 (size=12301) 2024-11-17T03:16:17,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/cb30e77b753b4cada9f7753b39c6f394 2024-11-17T03:16:17,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/62bef9568f7342f5bd027bb0eb9ed50a is 50, key is test_row_0/C:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:17,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742199_1375 (size=12301) 2024-11-17T03:16:17,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/62bef9568f7342f5bd027bb0eb9ed50a 2024-11-17T03:16:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/b0d48327c58a4a6c8b0fdefcd590c324 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324 2024-11-17T03:16:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813437407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813437407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324, entries=200, sequenceid=575, filesize=39.0 K 2024-11-17T03:16:17,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/cb30e77b753b4cada9f7753b39c6f394 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/cb30e77b753b4cada9f7753b39c6f394 2024-11-17T03:16:17,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/cb30e77b753b4cada9f7753b39c6f394, entries=150, sequenceid=575, filesize=12.0 K 2024-11-17T03:16:17,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813437427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813437437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/62bef9568f7342f5bd027bb0eb9ed50a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/62bef9568f7342f5bd027bb0eb9ed50a 2024-11-17T03:16:17,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/62bef9568f7342f5bd027bb0eb9ed50a, entries=150, sequenceid=575, filesize=12.0 K 2024-11-17T03:16:17,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for aa45336f99df582886fbc013393f2fd9 in 740ms, sequenceid=575, compaction requested=true 2024-11-17T03:16:17,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:17,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:17,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:17,465 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:17,465 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:17,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:17,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:17,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:17,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:17,467 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:17,467 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:17,467 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,467 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/caaaafe2ff3e4a8bbb54fbc9e579ae73, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9c93f9188c1140d9b8d48ece2e1be219, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/90d3611df93a41a4bf626dbd3cbca5ef, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/cb30e77b753b4cada9f7753b39c6f394] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=48.5 K 2024-11-17T03:16:17,468 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting caaaafe2ff3e4a8bbb54fbc9e579ae73, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731813374021 2024-11-17T03:16:17,468 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c93f9188c1140d9b8d48ece2e1be219, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731813374117 2024-11-17T03:16:17,468 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:17,468 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:17,468 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,469 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/736a2cc22db6419cb5c014cdae47a8d5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=139.5 K 2024-11-17T03:16:17,469 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,469 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/736a2cc22db6419cb5c014cdae47a8d5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324] 2024-11-17T03:16:17,469 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 736a2cc22db6419cb5c014cdae47a8d5, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731813374021 2024-11-17T03:16:17,469 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d3611df93a41a4bf626dbd3cbca5ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731813375308 2024-11-17T03:16:17,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6332baba009405e97c41f064b22b84b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731813374105 2024-11-17T03:16:17,470 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting cb30e77b753b4cada9f7753b39c6f394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731813375993 2024-11-17T03:16:17,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:17,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb2af504ba84e70a57fb4c8f50fb736, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731813375308 2024-11-17T03:16:17,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:16:17,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:17,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:17,471 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d48327c58a4a6c8b0fdefcd590c324, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731813375993 2024-11-17T03:16:17,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:17,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:17,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:17,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:17,497 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#315 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:17,498 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/8e3faead091245c0823f9f937dd4e5a9 is 50, key is test_row_0/B:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:17,506 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:17,530 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d8b859548af84dce9bf2c559bfe7a91e_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:17,533 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d8b859548af84dce9bf2c559bfe7a91e_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:17,533 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d8b859548af84dce9bf2c559bfe7a91e_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:17,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111778315527abf04cc59c7a80261b513077_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813377461/Put/seqid=0 2024-11-17T03:16:17,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742200_1376 (size=12915) 2024-11-17T03:16:17,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742201_1377 (size=4469) 2024-11-17T03:16:17,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742202_1378 (size=14994) 2024-11-17T03:16:17,640 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:17,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111778315527abf04cc59c7a80261b513077_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111778315527abf04cc59c7a80261b513077_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:17,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/9b1a6f8892dc40b197c44d86e3e805e7, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:17,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/9b1a6f8892dc40b197c44d86e3e805e7 is 175, key is test_row_0/A:col10/1731813377461/Put/seqid=0 2024-11-17T03:16:17,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742203_1379 (size=39949) 2024-11-17T03:16:17,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=590, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/9b1a6f8892dc40b197c44d86e3e805e7 2024-11-17T03:16:17,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/863d38480e5849ef93594afdccee3934 is 50, key is test_row_0/B:col10/1731813377461/Put/seqid=0 2024-11-17T03:16:17,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742204_1380 (size=12301) 2024-11-17T03:16:17,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/863d38480e5849ef93594afdccee3934 2024-11-17T03:16:17,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813437741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff is 50, key is test_row_0/C:col10/1731813377461/Put/seqid=0 2024-11-17T03:16:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:17,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742205_1381 (size=12301) 2024-11-17T03:16:17,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813437857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:17,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813437927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813437928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813437957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813437958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:17,979 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/8e3faead091245c0823f9f937dd4e5a9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/8e3faead091245c0823f9f937dd4e5a9 2024-11-17T03:16:17,988 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into 8e3faead091245c0823f9f937dd4e5a9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:17,988 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:17,988 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=12, startTime=1731813377465; duration=0sec 2024-11-17T03:16:17,988 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:17,988 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:17,988 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:17,989 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:17,989 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:17,989 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:17,989 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8a6eed3db29d4b8e84f6405170cd114c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/662b161e0aa0495b9ccdcb45c1eaf060, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/42b1fab13f1d44c9acf0ee22ae299cc1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/62bef9568f7342f5bd027bb0eb9ed50a] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=48.5 K 2024-11-17T03:16:17,990 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6eed3db29d4b8e84f6405170cd114c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731813374021 2024-11-17T03:16:17,990 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 662b161e0aa0495b9ccdcb45c1eaf060, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731813374117 2024-11-17T03:16:17,994 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b1fab13f1d44c9acf0ee22ae299cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731813375308 2024-11-17T03:16:17,997 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 62bef9568f7342f5bd027bb0eb9ed50a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731813375993 2024-11-17T03:16:18,006 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#320 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:18,006 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/9a3039c4493d41878d40abee412f1599 is 50, key is test_row_0/C:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:18,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742206_1382 (size=12915) 2024-11-17T03:16:18,021 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#316 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:18,021 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f2103e250fda424781dfe49fc78a95c6 is 175, key is test_row_0/A:col10/1731813375993/Put/seqid=0 2024-11-17T03:16:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742207_1383 (size=31869) 2024-11-17T03:16:18,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:18,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:18,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:18,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813438087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff 2024-11-17T03:16:18,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/9b1a6f8892dc40b197c44d86e3e805e7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7 2024-11-17T03:16:18,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7, entries=200, sequenceid=590, filesize=39.0 K 2024-11-17T03:16:18,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/863d38480e5849ef93594afdccee3934 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/863d38480e5849ef93594afdccee3934 2024-11-17T03:16:18,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/863d38480e5849ef93594afdccee3934, entries=150, sequenceid=590, filesize=12.0 K 2024-11-17T03:16:18,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff 2024-11-17T03:16:18,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff, entries=150, sequenceid=590, filesize=12.0 K 2024-11-17T03:16:18,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aa45336f99df582886fbc013393f2fd9 in 735ms, sequenceid=590, compaction requested=false 2024-11-17T03:16:18,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:18,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T03:16:18,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:18,213 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:16:18,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:18,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a641b69b4fe84c5db915575f19cbbdac_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813377739/Put/seqid=0 2024-11-17T03:16:18,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742208_1384 (size=12454) 2024-11-17T03:16:18,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:18,231 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a641b69b4fe84c5db915575f19cbbdac_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a641b69b4fe84c5db915575f19cbbdac_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e0b66360f2f44bc9a62e5da2f50a3e10, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e0b66360f2f44bc9a62e5da2f50a3e10 is 175, key is test_row_0/A:col10/1731813377739/Put/seqid=0 2024-11-17T03:16:18,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742209_1385 (size=31255) 2024-11-17T03:16:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:18,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:18,417 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/9a3039c4493d41878d40abee412f1599 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/9a3039c4493d41878d40abee412f1599 2024-11-17T03:16:18,420 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into 9a3039c4493d41878d40abee412f1599(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:18,420 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:18,420 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=12, startTime=1731813377466; duration=0sec 2024-11-17T03:16:18,420 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:18,420 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:18,438 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/f2103e250fda424781dfe49fc78a95c6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f2103e250fda424781dfe49fc78a95c6 2024-11-17T03:16:18,441 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into f2103e250fda424781dfe49fc78a95c6(size=31.1 K), total size for store is 70.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:18,441 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:18,441 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=12, startTime=1731813377465; duration=0sec 2024-11-17T03:16:18,441 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:18,441 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813438492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813438608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,643 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=608, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e0b66360f2f44bc9a62e5da2f50a3e10 2024-11-17T03:16:18,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/77c3632ad9944d3198d3edcc75d64b41 is 50, key is test_row_0/B:col10/1731813377739/Put/seqid=0 2024-11-17T03:16:18,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742210_1386 (size=12301) 2024-11-17T03:16:18,698 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/77c3632ad9944d3198d3edcc75d64b41 2024-11-17T03:16:18,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/95d628db9c23403e8ac8e75281b7e820 is 50, key is test_row_0/C:col10/1731813377739/Put/seqid=0 2024-11-17T03:16:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:18,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742211_1387 (size=12301) 2024-11-17T03:16:18,778 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/95d628db9c23403e8ac8e75281b7e820 2024-11-17T03:16:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/e0b66360f2f44bc9a62e5da2f50a3e10 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10 2024-11-17T03:16:18,797 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10, entries=150, sequenceid=608, filesize=30.5 K 2024-11-17T03:16:18,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/77c3632ad9944d3198d3edcc75d64b41 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/77c3632ad9944d3198d3edcc75d64b41 2024-11-17T03:16:18,802 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/77c3632ad9944d3198d3edcc75d64b41, entries=150, sequenceid=608, filesize=12.0 K 2024-11-17T03:16:18,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/95d628db9c23403e8ac8e75281b7e820 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/95d628db9c23403e8ac8e75281b7e820 2024-11-17T03:16:18,814 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/95d628db9c23403e8ac8e75281b7e820, entries=150, sequenceid=608, filesize=12.0 K 2024-11-17T03:16:18,815 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for aa45336f99df582886fbc013393f2fd9 in 602ms, sequenceid=608, compaction requested=true 2024-11-17T03:16:18,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:18,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:18,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T03:16:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T03:16:18,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T03:16:18,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-17T03:16:18,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2050 sec 2024-11-17T03:16:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:18,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T03:16:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:18,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:18,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117022ca4245c764384bc839b8913ea85cf_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:18,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742212_1388 (size=14994) 2024-11-17T03:16:18,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:18,885 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117022ca4245c764384bc839b8913ea85cf_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117022ca4245c764384bc839b8913ea85cf_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:18,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/fa5b395d29fd406384b7ed12ca9b3b2c, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:18,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/fa5b395d29fd406384b7ed12ca9b3b2c is 175, key is test_row_0/A:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:18,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742213_1389 (size=39949) 2024-11-17T03:16:18,917 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=626, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/fa5b395d29fd406384b7ed12ca9b3b2c 2024-11-17T03:16:18,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/455aa273c2d84ad3aa74bafce1672da0 is 50, key is test_row_0/B:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:18,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742214_1390 (size=12301) 2024-11-17T03:16:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813438972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813438978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,992 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813438980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,994 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813438993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813438993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:18,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813439097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813439107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813439107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813439307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813439326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813439327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/455aa273c2d84ad3aa74bafce1672da0 2024-11-17T03:16:19,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/3bdecdd1d9074dffa7ea9399fbf8fc34 is 50, key is test_row_0/C:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:19,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742215_1391 (size=12301) 2024-11-17T03:16:19,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/3bdecdd1d9074dffa7ea9399fbf8fc34 2024-11-17T03:16:19,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/fa5b395d29fd406384b7ed12ca9b3b2c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c 2024-11-17T03:16:19,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c, entries=200, sequenceid=626, filesize=39.0 K 2024-11-17T03:16:19,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/455aa273c2d84ad3aa74bafce1672da0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/455aa273c2d84ad3aa74bafce1672da0 2024-11-17T03:16:19,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/455aa273c2d84ad3aa74bafce1672da0, entries=150, sequenceid=626, filesize=12.0 K 2024-11-17T03:16:19,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/3bdecdd1d9074dffa7ea9399fbf8fc34 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/3bdecdd1d9074dffa7ea9399fbf8fc34 2024-11-17T03:16:19,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/3bdecdd1d9074dffa7ea9399fbf8fc34, entries=150, sequenceid=626, filesize=12.0 K 2024-11-17T03:16:19,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for aa45336f99df582886fbc013393f2fd9 in 546ms, sequenceid=626, compaction requested=true 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:19,388 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:19,388 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:19,390 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:19,390 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:19,390 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:19,390 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:19,390 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:19,390 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:19,390 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f2103e250fda424781dfe49fc78a95c6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=139.7 K 2024-11-17T03:16:19,390 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/8e3faead091245c0823f9f937dd4e5a9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/863d38480e5849ef93594afdccee3934, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/77c3632ad9944d3198d3edcc75d64b41, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/455aa273c2d84ad3aa74bafce1672da0] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=48.7 K 2024-11-17T03:16:19,390 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:19,390 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f2103e250fda424781dfe49fc78a95c6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c] 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3faead091245c0823f9f937dd4e5a9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731813375993 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2103e250fda424781dfe49fc78a95c6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731813375993 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 863d38480e5849ef93594afdccee3934, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731813376727 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b1a6f8892dc40b197c44d86e3e805e7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731813376727 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0b66360f2f44bc9a62e5da2f50a3e10, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1731813377646 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c3632ad9944d3198d3edcc75d64b41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1731813377646 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5b395d29fd406384b7ed12ca9b3b2c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731813378465 2024-11-17T03:16:19,391 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 455aa273c2d84ad3aa74bafce1672da0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731813378470 2024-11-17T03:16:19,397 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:19,398 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/59e253c5d50c42b39455ba55de766bd4 is 50, key is test_row_0/B:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:19,399 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:19,401 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173a75e6d4031e43f5b28df9ddb5337db4_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:19,403 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173a75e6d4031e43f5b28df9ddb5337db4_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:19,403 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173a75e6d4031e43f5b28df9ddb5337db4_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:19,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742216_1392 (size=13051) 2024-11-17T03:16:19,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742217_1393 (size=4469) 2024-11-17T03:16:19,407 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#328 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:19,407 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/ef07381c038b4b77913a8d7b179b02dc is 175, key is test_row_0/A:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:19,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742218_1394 (size=32005) 2024-11-17T03:16:19,411 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/59e253c5d50c42b39455ba55de766bd4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/59e253c5d50c42b39455ba55de766bd4 2024-11-17T03:16:19,414 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into 59e253c5d50c42b39455ba55de766bd4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:19,414 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:19,414 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=12, startTime=1731813379388; duration=0sec 2024-11-17T03:16:19,414 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:19,414 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:19,414 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:19,415 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:19,415 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:19,415 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:19,415 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/9a3039c4493d41878d40abee412f1599, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/95d628db9c23403e8ac8e75281b7e820, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/3bdecdd1d9074dffa7ea9399fbf8fc34] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=48.7 K 2024-11-17T03:16:19,415 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a3039c4493d41878d40abee412f1599, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1731813375993 2024-11-17T03:16:19,416 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 036cf8b8f3074c03b5e6e4eb5cb3e4ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731813376727 2024-11-17T03:16:19,416 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d628db9c23403e8ac8e75281b7e820, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1731813377646 2024-11-17T03:16:19,416 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bdecdd1d9074dffa7ea9399fbf8fc34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731813378470 2024-11-17T03:16:19,426 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#329 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:19,426 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/2969735b76954b6fac3817d27bfe3057 is 50, key is test_row_0/C:col10/1731813378470/Put/seqid=0 2024-11-17T03:16:19,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742219_1395 (size=13051) 2024-11-17T03:16:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:19,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:16:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:19,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b370096398534a1886ae9a5244dd42f2_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813378989/Put/seqid=0 2024-11-17T03:16:19,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742220_1396 (size=14994) 2024-11-17T03:16:19,700 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:19,707 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b370096398534a1886ae9a5244dd42f2_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b370096398534a1886ae9a5244dd42f2_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:19,708 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/35131b883387462fb68107d39440f8c7, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:19,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/35131b883387462fb68107d39440f8c7 is 175, key is test_row_0/A:col10/1731813378989/Put/seqid=0 2024-11-17T03:16:19,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813439703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813439703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813439705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742221_1397 (size=39949) 2024-11-17T03:16:19,818 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/ef07381c038b4b77913a8d7b179b02dc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ef07381c038b4b77913a8d7b179b02dc 2024-11-17T03:16:19,828 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into ef07381c038b4b77913a8d7b179b02dc(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:19,828 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:19,828 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=12, startTime=1731813379388; duration=0sec 2024-11-17T03:16:19,828 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:19,828 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:19,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813439817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813439817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813439827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:19,850 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/2969735b76954b6fac3817d27bfe3057 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2969735b76954b6fac3817d27bfe3057 2024-11-17T03:16:19,855 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into 2969735b76954b6fac3817d27bfe3057(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:19,855 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:19,855 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=12, startTime=1731813379388; duration=0sec 2024-11-17T03:16:19,855 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:19,855 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813440037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813440037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813440048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=645, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/35131b883387462fb68107d39440f8c7 2024-11-17T03:16:20,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/ef18865b391b45eb9f8de379755c2db2 is 50, key is test_row_0/B:col10/1731813378989/Put/seqid=0 2024-11-17T03:16:20,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742222_1398 (size=12301) 2024-11-17T03:16:20,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/ef18865b391b45eb9f8de379755c2db2 2024-11-17T03:16:20,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/4ac8f78a0e0f457997a9836fc04cfa31 is 50, key is test_row_0/C:col10/1731813378989/Put/seqid=0 2024-11-17T03:16:20,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742223_1399 (size=12301) 2024-11-17T03:16:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813440347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813440357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813440367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/4ac8f78a0e0f457997a9836fc04cfa31 2024-11-17T03:16:20,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/35131b883387462fb68107d39440f8c7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7 2024-11-17T03:16:20,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7, entries=200, sequenceid=645, filesize=39.0 K 2024-11-17T03:16:20,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/ef18865b391b45eb9f8de379755c2db2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/ef18865b391b45eb9f8de379755c2db2 2024-11-17T03:16:20,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/ef18865b391b45eb9f8de379755c2db2, entries=150, sequenceid=645, filesize=12.0 K 2024-11-17T03:16:20,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/4ac8f78a0e0f457997a9836fc04cfa31 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4ac8f78a0e0f457997a9836fc04cfa31 2024-11-17T03:16:20,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4ac8f78a0e0f457997a9836fc04cfa31, entries=150, sequenceid=645, filesize=12.0 K 2024-11-17T03:16:20,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aa45336f99df582886fbc013393f2fd9 in 998ms, sequenceid=645, compaction requested=false 2024-11-17T03:16:20,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T03:16:20,778 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:20,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T03:16:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:20,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:20,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:20,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:20,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:16:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:20,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:20,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111750daba5e6f324cd7847bae51f2c992ab_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:20,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742224_1400 (size=14994) 2024-11-17T03:16:20,868 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:20,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111750daba5e6f324cd7847bae51f2c992ab_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111750daba5e6f324cd7847bae51f2c992ab_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:20,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/aa09a32dbe254921a78a01d84b8f72e8, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:20,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/aa09a32dbe254921a78a01d84b8f72e8 is 175, key is test_row_0/A:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:20,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742225_1401 (size=39949) 2024-11-17T03:16:20,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=662, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/aa09a32dbe254921a78a01d84b8f72e8 2024-11-17T03:16:20,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/72b47fce60c24935b9d496d3dc0267ac is 50, key is test_row_0/B:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:20,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742226_1402 (size=12301) 2024-11-17T03:16:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:20,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813440902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813440905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813440905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:20,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:20,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:20,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:20,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:20,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48304 deadline: 1731813440997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,005 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48292 deadline: 1731813441017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813441017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813441017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813441017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,026 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in aa45336f99df582886fbc013393f2fd9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:21,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813441237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/72b47fce60c24935b9d496d3dc0267ac 2024-11-17T03:16:21,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/8cebffccf1144a7f96bf1c8faeff29fc is 50, key is test_row_0/C:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:21,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742227_1403 (size=12301) 2024-11-17T03:16:21,380 INFO [Thread-1412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1412. 2024-11-17T03:16:21,380 DEBUG [Thread-1412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:21,380 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,381 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:21,384 INFO [Thread-1410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1410. 2024-11-17T03:16:21,384 DEBUG [Thread-1410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:21,384 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,384 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:21,389 INFO [Thread-1418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1418. 2024-11-17T03:16:21,389 DEBUG [Thread-1418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:21,389 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,389 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:21,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:21,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:21,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,391 INFO [Thread-1414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1414. 2024-11-17T03:16:21,391 DEBUG [Thread-1414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:21,391 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,391 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:21,400 INFO [Thread-1416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1416. 2024-11-17T03:16:21,400 DEBUG [Thread-1416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:21,400 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,401 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:21,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:21,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:21,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48244 deadline: 1731813441557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48268 deadline: 1731813441557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48250 deadline: 1731813441557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:21,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/8cebffccf1144a7f96bf1c8faeff29fc 2024-11-17T03:16:21,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:21,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:21,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:21,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/aa09a32dbe254921a78a01d84b8f72e8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8 2024-11-17T03:16:21,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8, entries=200, sequenceid=662, filesize=39.0 K 2024-11-17T03:16:21,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/72b47fce60c24935b9d496d3dc0267ac as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/72b47fce60c24935b9d496d3dc0267ac 2024-11-17T03:16:21,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/72b47fce60c24935b9d496d3dc0267ac, entries=150, sequenceid=662, filesize=12.0 K 2024-11-17T03:16:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/8cebffccf1144a7f96bf1c8faeff29fc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8cebffccf1144a7f96bf1c8faeff29fc 2024-11-17T03:16:21,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8cebffccf1144a7f96bf1c8faeff29fc, entries=150, sequenceid=662, filesize=12.0 K 2024-11-17T03:16:21,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aa45336f99df582886fbc013393f2fd9 in 861ms, sequenceid=662, compaction requested=true 2024-11-17T03:16:21,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:21,720 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:21,720 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aa45336f99df582886fbc013393f2fd9:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:21,720 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:21,720 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:21,720 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/A is initiating minor compaction (all files) 2024-11-17T03:16:21,720 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/B is initiating minor compaction (all files) 2024-11-17T03:16:21,720 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/B in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,720 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/A in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,721 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/59e253c5d50c42b39455ba55de766bd4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/ef18865b391b45eb9f8de379755c2db2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/72b47fce60c24935b9d496d3dc0267ac] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=36.8 K 2024-11-17T03:16:21,721 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ef07381c038b4b77913a8d7b179b02dc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=109.3 K 2024-11-17T03:16:21,721 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ef07381c038b4b77913a8d7b179b02dc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8] 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e253c5d50c42b39455ba55de766bd4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731813378470 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef07381c038b4b77913a8d7b179b02dc, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731813378470 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ef18865b391b45eb9f8de379755c2db2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731813378965 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35131b883387462fb68107d39440f8c7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731813378965 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b47fce60c24935b9d496d3dc0267ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731813379686 2024-11-17T03:16:21,721 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa09a32dbe254921a78a01d84b8f72e8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731813379686 2024-11-17T03:16:21,727 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:21,728 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:21,728 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/1dc67cfacbce42708be7acac8b959ed2 is 50, key is test_row_0/B:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:21,729 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117972495f0053a4b42a94d2458f755e4b9_aa45336f99df582886fbc013393f2fd9 store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:21,732 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117972495f0053a4b42a94d2458f755e4b9_aa45336f99df582886fbc013393f2fd9, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:21,732 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117972495f0053a4b42a94d2458f755e4b9_aa45336f99df582886fbc013393f2fd9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742228_1404 (size=13153) 2024-11-17T03:16:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742229_1405 (size=4469) 2024-11-17T03:16:21,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:21,850 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:21,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:21,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c4f83046c4bc4cc9b26b54a14ebd1f4d_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813380904/Put/seqid=0 2024-11-17T03:16:21,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742230_1406 (size=12454) 2024-11-17T03:16:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:22,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. as already flushing 2024-11-17T03:16:22,068 INFO [Thread-1405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1405. 2024-11-17T03:16:22,068 INFO [Thread-1401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1401. 2024-11-17T03:16:22,068 INFO [Thread-1403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1403. 2024-11-17T03:16:22,068 DEBUG [Thread-1405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:22,068 DEBUG [Thread-1405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:22,068 DEBUG [Thread-1403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:22,068 DEBUG [Thread-1401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:22,068 DEBUG [Thread-1403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:22,068 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:22,068 DEBUG [Thread-1405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:22,068 DEBUG [Thread-1403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:22,068 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:22,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:22,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:22,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:22,138 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/1dc67cfacbce42708be7acac8b959ed2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1dc67cfacbce42708be7acac8b959ed2 2024-11-17T03:16:22,139 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:22,140 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/48ccbcea433f46c9af54710b6366059f is 175, key is test_row_0/A:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:22,144 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/B of aa45336f99df582886fbc013393f2fd9 into 1dc67cfacbce42708be7acac8b959ed2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:22,145 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:22,145 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/B, priority=13, startTime=1731813381720; duration=0sec 2024-11-17T03:16:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742231_1407 (size=32107) 2024-11-17T03:16:22,145 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:22,145 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:B 2024-11-17T03:16:22,145 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:22,147 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:22,147 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): aa45336f99df582886fbc013393f2fd9/C is initiating minor compaction (all files) 2024-11-17T03:16:22,147 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aa45336f99df582886fbc013393f2fd9/C in TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:22,147 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2969735b76954b6fac3817d27bfe3057, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4ac8f78a0e0f457997a9836fc04cfa31, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8cebffccf1144a7f96bf1c8faeff29fc] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp, totalSize=36.8 K 2024-11-17T03:16:22,148 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2969735b76954b6fac3817d27bfe3057, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1731813378470 2024-11-17T03:16:22,148 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac8f78a0e0f457997a9836fc04cfa31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1731813378965 2024-11-17T03:16:22,148 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cebffccf1144a7f96bf1c8faeff29fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1731813379686 2024-11-17T03:16:22,151 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/48ccbcea433f46c9af54710b6366059f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/48ccbcea433f46c9af54710b6366059f 2024-11-17T03:16:22,155 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/A of aa45336f99df582886fbc013393f2fd9 into 48ccbcea433f46c9af54710b6366059f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:22,155 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:22,155 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/A, priority=13, startTime=1731813381719; duration=0sec 2024-11-17T03:16:22,155 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:22,155 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:A 2024-11-17T03:16:22,156 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aa45336f99df582886fbc013393f2fd9#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:22,156 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/94eebb39874e418aace85028dabe9080 is 50, key is test_row_0/C:col10/1731813380857/Put/seqid=0 2024-11-17T03:16:22,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742232_1408 (size=13153) 2024-11-17T03:16:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:22,267 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c4f83046c4bc4cc9b26b54a14ebd1f4d_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c4f83046c4bc4cc9b26b54a14ebd1f4d_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:22,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/d4742741c8a848efae9f0b8bbe05daf3, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:22,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/d4742741c8a848efae9f0b8bbe05daf3 is 175, key is test_row_0/A:col10/1731813380904/Put/seqid=0 2024-11-17T03:16:22,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742233_1409 (size=31255) 2024-11-17T03:16:22,571 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/94eebb39874e418aace85028dabe9080 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/94eebb39874e418aace85028dabe9080 2024-11-17T03:16:22,577 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aa45336f99df582886fbc013393f2fd9/C of aa45336f99df582886fbc013393f2fd9 into 94eebb39874e418aace85028dabe9080(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:22,577 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:22,577 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., storeName=aa45336f99df582886fbc013393f2fd9/C, priority=13, startTime=1731813381720; duration=0sec 2024-11-17T03:16:22,578 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:22,578 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aa45336f99df582886fbc013393f2fd9:C 2024-11-17T03:16:22,673 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=679, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/d4742741c8a848efae9f0b8bbe05daf3 2024-11-17T03:16:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/a307657213424b28b6c3cc87fdd6e2ec is 50, key is test_row_0/B:col10/1731813380904/Put/seqid=0 2024-11-17T03:16:22,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742234_1410 (size=12301) 2024-11-17T03:16:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:23,083 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/a307657213424b28b6c3cc87fdd6e2ec 2024-11-17T03:16:23,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/34eae1ac4b0744ce8375c8a2df9b0741 is 50, key is test_row_0/C:col10/1731813380904/Put/seqid=0 2024-11-17T03:16:23,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742235_1411 (size=12301) 2024-11-17T03:16:23,498 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/34eae1ac4b0744ce8375c8a2df9b0741 2024-11-17T03:16:23,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/d4742741c8a848efae9f0b8bbe05daf3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/d4742741c8a848efae9f0b8bbe05daf3 2024-11-17T03:16:23,513 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/d4742741c8a848efae9f0b8bbe05daf3, entries=150, sequenceid=679, filesize=30.5 K 2024-11-17T03:16:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/a307657213424b28b6c3cc87fdd6e2ec as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a307657213424b28b6c3cc87fdd6e2ec 2024-11-17T03:16:23,519 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a307657213424b28b6c3cc87fdd6e2ec, entries=150, sequenceid=679, filesize=12.0 K 2024-11-17T03:16:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/34eae1ac4b0744ce8375c8a2df9b0741 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/34eae1ac4b0744ce8375c8a2df9b0741 2024-11-17T03:16:23,523 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/34eae1ac4b0744ce8375c8a2df9b0741, entries=150, sequenceid=679, filesize=12.0 K 2024-11-17T03:16:23,523 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for aa45336f99df582886fbc013393f2fd9 in 1673ms, sequenceid=679, compaction requested=false 2024-11-17T03:16:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for aa45336f99df582886fbc013393f2fd9: 2024-11-17T03:16:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T03:16:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T03:16:23,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T03:16:23,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7440 sec 2024-11-17T03:16:23,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.7480 sec 2024-11-17T03:16:24,353 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T03:16:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T03:16:24,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:25,018 INFO [Thread-1407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1407. 2024-11-17T03:16:25,018 DEBUG [Thread-1407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:25,018 DEBUG [Thread-1407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:25,018 DEBUG [Thread-1407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:25,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:25,059 INFO [Thread-1399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1399. 2024-11-17T03:16:25,059 DEBUG [Thread-1399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:25,059 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:25,059 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:25,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 173 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 898 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2692 rows 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 896 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2687 rows 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 898 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2688 rows 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 894 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2681 rows 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 898 2024-11-17T03:16:25,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2690 rows 2024-11-17T03:16:25,059 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:16:25,060 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:16:25,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:25,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:25,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:25,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T03:16:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T03:16:25,063 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813385063"}]},"ts":"1731813385063"} 2024-11-17T03:16:25,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T03:16:25,064 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T03:16:25,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:16:25,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, UNASSIGN}] 2024-11-17T03:16:25,066 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, UNASSIGN 2024-11-17T03:16:25,067 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:25,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, UNASSIGN because future has completed 2024-11-17T03:16:25,068 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:16:25,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T03:16:25,220 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:25,220 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing aa45336f99df582886fbc013393f2fd9, disabling compactions & flushes 2024-11-17T03:16:25,221 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. after waiting 0 ms 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:25,221 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing aa45336f99df582886fbc013393f2fd9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=A 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=B 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aa45336f99df582886fbc013393f2fd9, store=C 2024-11-17T03:16:25,221 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:25,226 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117332464305c344e5d9dd35035987b2b0b_aa45336f99df582886fbc013393f2fd9 is 50, key is test_row_0/A:col10/1731813385057/Put/seqid=0 2024-11-17T03:16:25,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742236_1412 (size=12454) 2024-11-17T03:16:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T03:16:25,629 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:25,632 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117332464305c344e5d9dd35035987b2b0b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117332464305c344e5d9dd35035987b2b0b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:25,633 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/1cc3d5d02b2047cb8023790229825e6e, store: [table=TestAcidGuarantees family=A region=aa45336f99df582886fbc013393f2fd9] 2024-11-17T03:16:25,633 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/1cc3d5d02b2047cb8023790229825e6e is 175, key is test_row_0/A:col10/1731813385057/Put/seqid=0 2024-11-17T03:16:25,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742237_1413 (size=31255) 2024-11-17T03:16:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T03:16:26,037 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=690, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/1cc3d5d02b2047cb8023790229825e6e 2024-11-17T03:16:26,045 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/29910733b29e4be6a587eb1cdab5593e is 50, key is test_row_0/B:col10/1731813385057/Put/seqid=0 2024-11-17T03:16:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742238_1414 (size=12301) 2024-11-17T03:16:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T03:16:26,451 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/29910733b29e4be6a587eb1cdab5593e 2024-11-17T03:16:26,464 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/41bceb5eba5042369f79b2d5de8d224c is 50, key is test_row_0/C:col10/1731813385057/Put/seqid=0 2024-11-17T03:16:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742239_1415 (size=12301) 2024-11-17T03:16:26,870 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/41bceb5eba5042369f79b2d5de8d224c 2024-11-17T03:16:26,878 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/A/1cc3d5d02b2047cb8023790229825e6e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1cc3d5d02b2047cb8023790229825e6e 2024-11-17T03:16:26,881 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1cc3d5d02b2047cb8023790229825e6e, entries=150, sequenceid=690, filesize=30.5 K 2024-11-17T03:16:26,882 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/B/29910733b29e4be6a587eb1cdab5593e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/29910733b29e4be6a587eb1cdab5593e 2024-11-17T03:16:26,885 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/29910733b29e4be6a587eb1cdab5593e, entries=150, sequenceid=690, filesize=12.0 K 2024-11-17T03:16:26,886 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/.tmp/C/41bceb5eba5042369f79b2d5de8d224c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/41bceb5eba5042369f79b2d5de8d224c 2024-11-17T03:16:26,890 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/41bceb5eba5042369f79b2d5de8d224c, entries=150, sequenceid=690, filesize=12.0 K 2024-11-17T03:16:26,891 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aa45336f99df582886fbc013393f2fd9 in 1670ms, sequenceid=690, compaction requested=true 2024-11-17T03:16:26,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57959104ceec4173a70374d1a5738d0c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0b36f771421f4557a1fd9ab5ee8af3dd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1e1511e1f2f34621aa3199423613271a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/736a2cc22db6419cb5c014cdae47a8d5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f2103e250fda424781dfe49fc78a95c6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ef07381c038b4b77913a8d7b179b02dc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8] to archive 2024-11-17T03:16:26,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:16:26,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/be9facac0b7141a7a708b57c9b2b3f5e 2024-11-17T03:16:26,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/c5982a5b84dc4573b1b750ccf52a0d04 2024-11-17T03:16:26,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0ad7b165d4a64d96914979d33acf1615 2024-11-17T03:16:26,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57959104ceec4173a70374d1a5738d0c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57959104ceec4173a70374d1a5738d0c 2024-11-17T03:16:26,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2404b7abd2004bcc88c88cf9bbdcd97e 2024-11-17T03:16:26,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e643caf298d84a17b6a9ded307939958 2024-11-17T03:16:26,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f56e39ea563e400588c56be5fb8301f6 2024-11-17T03:16:26,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0b36f771421f4557a1fd9ab5ee8af3dd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/0b36f771421f4557a1fd9ab5ee8af3dd 2024-11-17T03:16:26,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/2d39201cb2944f08a7b0a581dd067d0a 2024-11-17T03:16:26,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/034e8cf737e14910b8fb1c983374c029 2024-11-17T03:16:26,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/70494319ed0945558a84d7e0a7f4b8ba 2024-11-17T03:16:26,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1e1511e1f2f34621aa3199423613271a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1e1511e1f2f34621aa3199423613271a 2024-11-17T03:16:26,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/57e6082e93804aee8ef28d07749aaf43 2024-11-17T03:16:26,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/736a2cc22db6419cb5c014cdae47a8d5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/736a2cc22db6419cb5c014cdae47a8d5 2024-11-17T03:16:26,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/4b0b3603b7194c87bef2ea3cfe9a34e1 2024-11-17T03:16:26,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f6332baba009405e97c41f064b22b84b 2024-11-17T03:16:26,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ecb2af504ba84e70a57fb4c8f50fb736 2024-11-17T03:16:26,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/b0d48327c58a4a6c8b0fdefcd590c324 2024-11-17T03:16:26,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f2103e250fda424781dfe49fc78a95c6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/f2103e250fda424781dfe49fc78a95c6 2024-11-17T03:16:26,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/9b1a6f8892dc40b197c44d86e3e805e7 2024-11-17T03:16:26,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/e0b66360f2f44bc9a62e5da2f50a3e10 2024-11-17T03:16:26,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/fa5b395d29fd406384b7ed12ca9b3b2c 2024-11-17T03:16:26,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ef07381c038b4b77913a8d7b179b02dc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/ef07381c038b4b77913a8d7b179b02dc 2024-11-17T03:16:26,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/35131b883387462fb68107d39440f8c7 2024-11-17T03:16:26,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/aa09a32dbe254921a78a01d84b8f72e8 2024-11-17T03:16:26,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/345d9496f0fd48b7a887a4601970fc5e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9f6ae07c850049318adc641c6aa566d7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/39994be440604a9f9d0d818477d5adbc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/88a8dc304a8741268788d3e2a483411b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4f14dc35044a477eb9400f9825b0007a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a64a3768e38343f080eb7f8379b3ba45, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/dcb463fe3b114abcb2cd13c04e0d96ea, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1c5d937d947c425c9175aa4de878cd3b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/7942d7a83e584d248d0ea0b7e9610094, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/536290bd047b4f1fb286d4239b1d4864, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4fe083bde06246fbb2a3ae7cf856d6f4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/897730ae150d4e6fb99a981fe14f4226, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/5292f694648847f3a487ad9e56972e9a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/caaaafe2ff3e4a8bbb54fbc9e579ae73, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/3d84798fc87d448ab6684bdb751fb32d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9c93f9188c1140d9b8d48ece2e1be219, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/90d3611df93a41a4bf626dbd3cbca5ef, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/8e3faead091245c0823f9f937dd4e5a9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/cb30e77b753b4cada9f7753b39c6f394, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/863d38480e5849ef93594afdccee3934, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/77c3632ad9944d3198d3edcc75d64b41, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/59e253c5d50c42b39455ba55de766bd4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/455aa273c2d84ad3aa74bafce1672da0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/ef18865b391b45eb9f8de379755c2db2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/72b47fce60c24935b9d496d3dc0267ac] to archive 2024-11-17T03:16:26,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:16:26,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/345d9496f0fd48b7a887a4601970fc5e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/345d9496f0fd48b7a887a4601970fc5e 2024-11-17T03:16:26,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9f6ae07c850049318adc641c6aa566d7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9f6ae07c850049318adc641c6aa566d7 2024-11-17T03:16:26,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/39994be440604a9f9d0d818477d5adbc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/39994be440604a9f9d0d818477d5adbc 2024-11-17T03:16:26,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/88a8dc304a8741268788d3e2a483411b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/88a8dc304a8741268788d3e2a483411b 2024-11-17T03:16:26,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4f14dc35044a477eb9400f9825b0007a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4f14dc35044a477eb9400f9825b0007a 2024-11-17T03:16:26,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a64a3768e38343f080eb7f8379b3ba45 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a64a3768e38343f080eb7f8379b3ba45 2024-11-17T03:16:26,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/dcb463fe3b114abcb2cd13c04e0d96ea to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/dcb463fe3b114abcb2cd13c04e0d96ea 2024-11-17T03:16:26,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1c5d937d947c425c9175aa4de878cd3b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1c5d937d947c425c9175aa4de878cd3b 2024-11-17T03:16:26,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/7942d7a83e584d248d0ea0b7e9610094 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/7942d7a83e584d248d0ea0b7e9610094 2024-11-17T03:16:26,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/536290bd047b4f1fb286d4239b1d4864 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/536290bd047b4f1fb286d4239b1d4864 2024-11-17T03:16:26,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4fe083bde06246fbb2a3ae7cf856d6f4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/4fe083bde06246fbb2a3ae7cf856d6f4 2024-11-17T03:16:26,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/897730ae150d4e6fb99a981fe14f4226 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/897730ae150d4e6fb99a981fe14f4226 2024-11-17T03:16:26,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/5292f694648847f3a487ad9e56972e9a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/5292f694648847f3a487ad9e56972e9a 2024-11-17T03:16:26,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/caaaafe2ff3e4a8bbb54fbc9e579ae73 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/caaaafe2ff3e4a8bbb54fbc9e579ae73 2024-11-17T03:16:26,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/3d84798fc87d448ab6684bdb751fb32d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/3d84798fc87d448ab6684bdb751fb32d 2024-11-17T03:16:26,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9c93f9188c1140d9b8d48ece2e1be219 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/9c93f9188c1140d9b8d48ece2e1be219 2024-11-17T03:16:26,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/90d3611df93a41a4bf626dbd3cbca5ef to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/90d3611df93a41a4bf626dbd3cbca5ef 2024-11-17T03:16:26,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/8e3faead091245c0823f9f937dd4e5a9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/8e3faead091245c0823f9f937dd4e5a9 2024-11-17T03:16:26,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/cb30e77b753b4cada9f7753b39c6f394 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/cb30e77b753b4cada9f7753b39c6f394 2024-11-17T03:16:26,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/863d38480e5849ef93594afdccee3934 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/863d38480e5849ef93594afdccee3934 2024-11-17T03:16:26,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/77c3632ad9944d3198d3edcc75d64b41 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/77c3632ad9944d3198d3edcc75d64b41 2024-11-17T03:16:26,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/59e253c5d50c42b39455ba55de766bd4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/59e253c5d50c42b39455ba55de766bd4 2024-11-17T03:16:26,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/455aa273c2d84ad3aa74bafce1672da0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/455aa273c2d84ad3aa74bafce1672da0 2024-11-17T03:16:26,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/ef18865b391b45eb9f8de379755c2db2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/ef18865b391b45eb9f8de379755c2db2 2024-11-17T03:16:26,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/72b47fce60c24935b9d496d3dc0267ac to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/72b47fce60c24935b9d496d3dc0267ac 2024-11-17T03:16:26,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/e875be0169b24b4099b3075785c0aaa5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/51a7bbedf82240d9a435ef416b8419e1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/1c93c3f1a7694d63862e3c435bba4e42, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2945608ca2664d448e0534cdfd83ce84, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/0a5ff04efa50496b94b0f9686add047e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/69d51ca3e8ac461ea1a6a96114ca5e1b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/80eb0ccd6e994275ba5d72f9dfdddc98, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/f10d103ee1314a388ea7828344344208, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/5d139d4f14b6497ebd3a7ef7a4f540c5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d7374d2976d44d89aad10e4fc2c6cf7e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/b9ee84e567ba454ea028c92b73f38915, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4d6b56cb26114f2195b78bfd9586de6c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/da49435a96304600a17c256359d69d78, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8a6eed3db29d4b8e84f6405170cd114c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d3b29b2a71414dfdb2df57cbd274b974, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/662b161e0aa0495b9ccdcb45c1eaf060, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/42b1fab13f1d44c9acf0ee22ae299cc1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/9a3039c4493d41878d40abee412f1599, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/62bef9568f7342f5bd027bb0eb9ed50a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/95d628db9c23403e8ac8e75281b7e820, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2969735b76954b6fac3817d27bfe3057, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/3bdecdd1d9074dffa7ea9399fbf8fc34, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4ac8f78a0e0f457997a9836fc04cfa31, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8cebffccf1144a7f96bf1c8faeff29fc] to archive 2024-11-17T03:16:26,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:16:26,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/e875be0169b24b4099b3075785c0aaa5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/e875be0169b24b4099b3075785c0aaa5 2024-11-17T03:16:26,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/51a7bbedf82240d9a435ef416b8419e1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/51a7bbedf82240d9a435ef416b8419e1 2024-11-17T03:16:26,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/1c93c3f1a7694d63862e3c435bba4e42 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/1c93c3f1a7694d63862e3c435bba4e42 2024-11-17T03:16:26,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2945608ca2664d448e0534cdfd83ce84 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2945608ca2664d448e0534cdfd83ce84 2024-11-17T03:16:26,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/0a5ff04efa50496b94b0f9686add047e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/0a5ff04efa50496b94b0f9686add047e 2024-11-17T03:16:26,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/69d51ca3e8ac461ea1a6a96114ca5e1b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/69d51ca3e8ac461ea1a6a96114ca5e1b 2024-11-17T03:16:26,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/80eb0ccd6e994275ba5d72f9dfdddc98 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/80eb0ccd6e994275ba5d72f9dfdddc98 2024-11-17T03:16:26,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/f10d103ee1314a388ea7828344344208 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/f10d103ee1314a388ea7828344344208 2024-11-17T03:16:26,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/5d139d4f14b6497ebd3a7ef7a4f540c5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/5d139d4f14b6497ebd3a7ef7a4f540c5 2024-11-17T03:16:26,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d7374d2976d44d89aad10e4fc2c6cf7e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d7374d2976d44d89aad10e4fc2c6cf7e 2024-11-17T03:16:26,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/b9ee84e567ba454ea028c92b73f38915 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/b9ee84e567ba454ea028c92b73f38915 2024-11-17T03:16:26,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4d6b56cb26114f2195b78bfd9586de6c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4d6b56cb26114f2195b78bfd9586de6c 2024-11-17T03:16:26,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/da49435a96304600a17c256359d69d78 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/da49435a96304600a17c256359d69d78 2024-11-17T03:16:26,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8a6eed3db29d4b8e84f6405170cd114c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8a6eed3db29d4b8e84f6405170cd114c 2024-11-17T03:16:26,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d3b29b2a71414dfdb2df57cbd274b974 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/d3b29b2a71414dfdb2df57cbd274b974 2024-11-17T03:16:26,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/662b161e0aa0495b9ccdcb45c1eaf060 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/662b161e0aa0495b9ccdcb45c1eaf060 2024-11-17T03:16:26,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/42b1fab13f1d44c9acf0ee22ae299cc1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/42b1fab13f1d44c9acf0ee22ae299cc1 2024-11-17T03:16:26,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/9a3039c4493d41878d40abee412f1599 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/9a3039c4493d41878d40abee412f1599 2024-11-17T03:16:26,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/62bef9568f7342f5bd027bb0eb9ed50a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/62bef9568f7342f5bd027bb0eb9ed50a 2024-11-17T03:16:26,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/036cf8b8f3074c03b5e6e4eb5cb3e4ff 2024-11-17T03:16:26,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/95d628db9c23403e8ac8e75281b7e820 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/95d628db9c23403e8ac8e75281b7e820 2024-11-17T03:16:26,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2969735b76954b6fac3817d27bfe3057 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/2969735b76954b6fac3817d27bfe3057 2024-11-17T03:16:26,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/3bdecdd1d9074dffa7ea9399fbf8fc34 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/3bdecdd1d9074dffa7ea9399fbf8fc34 2024-11-17T03:16:26,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4ac8f78a0e0f457997a9836fc04cfa31 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/4ac8f78a0e0f457997a9836fc04cfa31 2024-11-17T03:16:26,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8cebffccf1144a7f96bf1c8faeff29fc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/8cebffccf1144a7f96bf1c8faeff29fc 2024-11-17T03:16:26,954 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/recovered.edits/693.seqid, newMaxSeqId=693, maxSeqId=4 2024-11-17T03:16:26,954 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. 2024-11-17T03:16:26,954 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for aa45336f99df582886fbc013393f2fd9: Waiting for close lock at 1731813385221Running coprocessor pre-close hooks at 1731813385221Disabling compacts and flushes for region at 1731813385221Disabling writes for close at 1731813385221Obtaining lock to block concurrent updates at 1731813385221Preparing flush snapshotting stores in aa45336f99df582886fbc013393f2fd9 at 1731813385221Finished memstore snapshotting TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731813385221Flushing stores of TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9. at 1731813385222 (+1 ms)Flushing aa45336f99df582886fbc013393f2fd9/A: creating writer at 1731813385222Flushing mob file aa45336f99df582886fbc013393f2fd9/A: appending metadata at 1731813385226 (+4 ms)Flushing mob file aa45336f99df582886fbc013393f2fd9/A: closing flushed file at 1731813385226Flushing aa45336f99df582886fbc013393f2fd9/A: appending metadata at 1731813385633 (+407 ms)Flushing aa45336f99df582886fbc013393f2fd9/A: closing flushed file at 1731813385633Flushing aa45336f99df582886fbc013393f2fd9/B: creating writer at 1731813386041 (+408 ms)Flushing aa45336f99df582886fbc013393f2fd9/B: appending metadata at 1731813386045 (+4 ms)Flushing aa45336f99df582886fbc013393f2fd9/B: closing flushed file at 1731813386045Flushing aa45336f99df582886fbc013393f2fd9/C: creating writer at 1731813386460 (+415 ms)Flushing aa45336f99df582886fbc013393f2fd9/C: appending metadata at 1731813386464 (+4 ms)Flushing aa45336f99df582886fbc013393f2fd9/C: closing flushed file at 1731813386464Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@493120da: reopening flushed file at 1731813386877 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@345a3c29: reopening flushed file at 1731813386881 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b1dcb9: reopening flushed file at 1731813386886 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aa45336f99df582886fbc013393f2fd9 in 1670ms, sequenceid=690, compaction requested=true at 1731813386891 (+5 ms)Writing region close event to WAL at 1731813386952 (+61 ms)Running coprocessor post-close hooks at 1731813386954 (+2 ms)Closed at 1731813386954 2024-11-17T03:16:26,956 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:26,956 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=aa45336f99df582886fbc013393f2fd9, regionState=CLOSED 2024-11-17T03:16:26,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:26,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T03:16:26,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure aa45336f99df582886fbc013393f2fd9, server=695c90f4bb18,38753,1731813266381 in 1.8900 sec 2024-11-17T03:16:26,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-17T03:16:26,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aa45336f99df582886fbc013393f2fd9, UNASSIGN in 1.8930 sec 2024-11-17T03:16:26,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T03:16:26,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8950 sec 2024-11-17T03:16:26,962 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813386962"}]},"ts":"1731813386962"} 2024-11-17T03:16:26,963 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T03:16:26,963 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T03:16:26,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9030 sec 2024-11-17T03:16:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T03:16:27,207 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:27,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T03:16:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,209 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T03:16:27,209 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,211 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,212 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/recovered.edits] 2024-11-17T03:16:27,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1cc3d5d02b2047cb8023790229825e6e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/1cc3d5d02b2047cb8023790229825e6e 2024-11-17T03:16:27,215 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/48ccbcea433f46c9af54710b6366059f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/48ccbcea433f46c9af54710b6366059f 2024-11-17T03:16:27,217 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/d4742741c8a848efae9f0b8bbe05daf3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/A/d4742741c8a848efae9f0b8bbe05daf3 2024-11-17T03:16:27,219 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1dc67cfacbce42708be7acac8b959ed2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/1dc67cfacbce42708be7acac8b959ed2 2024-11-17T03:16:27,220 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/29910733b29e4be6a587eb1cdab5593e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/29910733b29e4be6a587eb1cdab5593e 2024-11-17T03:16:27,222 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a307657213424b28b6c3cc87fdd6e2ec to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/B/a307657213424b28b6c3cc87fdd6e2ec 2024-11-17T03:16:27,224 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/34eae1ac4b0744ce8375c8a2df9b0741 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/34eae1ac4b0744ce8375c8a2df9b0741 2024-11-17T03:16:27,226 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/41bceb5eba5042369f79b2d5de8d224c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/41bceb5eba5042369f79b2d5de8d224c 2024-11-17T03:16:27,227 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/94eebb39874e418aace85028dabe9080 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/C/94eebb39874e418aace85028dabe9080 2024-11-17T03:16:27,229 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/recovered.edits/693.seqid to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9/recovered.edits/693.seqid 2024-11-17T03:16:27,230 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,230 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T03:16:27,230 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T03:16:27,231 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T03:16:27,234 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117022ca4245c764384bc839b8913ea85cf_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117022ca4245c764384bc839b8913ea85cf_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,235 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172bbbaaadca534895b7d4603fcf7ffa41_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172bbbaaadca534895b7d4603fcf7ffa41_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,235 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111732b9dab8fbb046d5aeea7750b8eea0b9_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111732b9dab8fbb046d5aeea7750b8eea0b9_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,236 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117332464305c344e5d9dd35035987b2b0b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117332464305c344e5d9dd35035987b2b0b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,237 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117423336cc59624960bdded85f6820671b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117423336cc59624960bdded85f6820671b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,238 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111750daba5e6f324cd7847bae51f2c992ab_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111750daba5e6f324cd7847bae51f2c992ab_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,239 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175508fbc1bc1146f1baa540279cfe0158_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175508fbc1bc1146f1baa540279cfe0158_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,240 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175a0c22e22fe74ef08af7967e3c927551_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175a0c22e22fe74ef08af7967e3c927551_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,241 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b50fa86c3a14f03ad1d619c1fa6c181_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175b50fa86c3a14f03ad1d619c1fa6c181_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,242 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117737386070949459ca1461c6355a3710b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117737386070949459ca1461c6355a3710b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,243 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111778315527abf04cc59c7a80261b513077_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111778315527abf04cc59c7a80261b513077_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,244 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111790aff0ab9ced43b58bff8384d6c2316d_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111790aff0ab9ced43b58bff8384d6c2316d_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,245 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a641b69b4fe84c5db915575f19cbbdac_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a641b69b4fe84c5db915575f19cbbdac_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,245 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aa634fab5d0e47b3b3e09b1f43692107_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aa634fab5d0e47b3b3e09b1f43692107_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,246 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b370096398534a1886ae9a5244dd42f2_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b370096398534a1886ae9a5244dd42f2_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,247 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c4f83046c4bc4cc9b26b54a14ebd1f4d_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c4f83046c4bc4cc9b26b54a14ebd1f4d_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,248 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cbe58d804b514c4180c60a38bf74de7b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cbe58d804b514c4180c60a38bf74de7b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,249 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d1cc1ddab0c249568fe9f8787ad4884b_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d1cc1ddab0c249568fe9f8787ad4884b_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,250 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117de305d8165cf485982fb60580ecbf02c_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117de305d8165cf485982fb60580ecbf02c_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,251 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ecff41c844104cd691f3b604dd010657_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ecff41c844104cd691f3b604dd010657_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,251 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fc1476fce8e64888b93e6b458f716b2f_aa45336f99df582886fbc013393f2fd9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117fc1476fce8e64888b93e6b458f716b2f_aa45336f99df582886fbc013393f2fd9 2024-11-17T03:16:27,252 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T03:16:27,253 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,255 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T03:16:27,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T03:16:27,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,257 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T03:16:27,258 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731813387257"}]},"ts":"9223372036854775807"} 2024-11-17T03:16:27,259 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T03:16:27,259 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aa45336f99df582886fbc013393f2fd9, NAME => 'TestAcidGuarantees,,1731813358256.aa45336f99df582886fbc013393f2fd9.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T03:16:27,259 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T03:16:27,259 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731813387259"}]},"ts":"9223372036854775807"} 2024-11-17T03:16:27,261 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T03:16:27,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-17T03:16:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T03:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T03:16:27,319 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:27,325 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_570995009_22 at /127.0.0.1:35480 [Waiting for operation #2075] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-131647061_22 at /127.0.0.1:59872 [Waiting for operation #1642] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_570995009_22 at /127.0.0.1:59870 [Waiting for operation #1677] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-131647061_22 at /127.0.0.1:56100 [Waiting for operation #1821] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 467), ProcessCount=11 (was 11), AvailableMemoryMB=1079 (was 1655) 2024-11-17T03:16:27,334 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=1078 2024-11-17T03:16:27,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:16:27,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:27,338 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T03:16:27,338 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:27,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-17T03:16:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T03:16:27,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T03:16:27,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742240_1416 (size=1061) 2024-11-17T03:16:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T03:16:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T03:16:27,747 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:16:27,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742241_1417 (size=53) 2024-11-17T03:16:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T03:16:28,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:16:28,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ed49cc07eb87ee1fb278a842d5aee3f6, disabling compactions & flushes 2024-11-17T03:16:28,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. after waiting 0 ms 2024-11-17T03:16:28,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,153 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,153 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed49cc07eb87ee1fb278a842d5aee3f6: Waiting for close lock at 1731813388153Disabling compacts and flushes for region at 1731813388153Disabling writes for close at 1731813388153Writing region close event to WAL at 1731813388153Closed at 1731813388153 2024-11-17T03:16:28,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T03:16:28,155 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731813388155"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731813388155"}]},"ts":"1731813388155"} 2024-11-17T03:16:28,158 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T03:16:28,159 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T03:16:28,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813388159"}]},"ts":"1731813388159"} 2024-11-17T03:16:28,161 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T03:16:28,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, ASSIGN}] 2024-11-17T03:16:28,163 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, ASSIGN 2024-11-17T03:16:28,164 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, ASSIGN; state=OFFLINE, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=false 2024-11-17T03:16:28,316 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=ed49cc07eb87ee1fb278a842d5aee3f6, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:28,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, ASSIGN because future has completed 2024-11-17T03:16:28,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T03:16:28,482 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,482 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:16:28,484 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,484 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:16:28,484 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,484 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,485 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,486 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:28,486 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed49cc07eb87ee1fb278a842d5aee3f6 columnFamilyName A 2024-11-17T03:16:28,487 DEBUG [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:28,487 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.HStore(327): Store=ed49cc07eb87ee1fb278a842d5aee3f6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:28,487 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,488 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:28,488 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed49cc07eb87ee1fb278a842d5aee3f6 columnFamilyName B 2024-11-17T03:16:28,488 DEBUG [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:28,488 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.HStore(327): Store=ed49cc07eb87ee1fb278a842d5aee3f6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:28,488 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,489 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:28,489 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed49cc07eb87ee1fb278a842d5aee3f6 columnFamilyName C 2024-11-17T03:16:28,489 DEBUG [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:28,490 INFO [StoreOpener-ed49cc07eb87ee1fb278a842d5aee3f6-1 {}] regionserver.HStore(327): Store=ed49cc07eb87ee1fb278a842d5aee3f6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:28,490 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,490 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,490 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,491 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,491 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,491 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,492 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:16:28,493 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,494 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:16:28,495 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened ed49cc07eb87ee1fb278a842d5aee3f6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65530032, jitterRate=-0.023526430130004883}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:16:28,495 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:28,495 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for ed49cc07eb87ee1fb278a842d5aee3f6: Running coprocessor pre-open hook at 1731813388484Writing region info on filesystem at 1731813388484Initializing all the Stores at 1731813388485 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813388485Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813388485Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813388485Cleaning up temporary data from old regions at 1731813388491 (+6 ms)Running coprocessor post-open hooks at 1731813388495 (+4 ms)Region opened successfully at 1731813388495 2024-11-17T03:16:28,496 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., pid=118, masterSystemTime=1731813388474 2024-11-17T03:16:28,497 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,497 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:28,498 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=ed49cc07eb87ee1fb278a842d5aee3f6, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:28,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:28,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T03:16:28,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 in 179 msec 2024-11-17T03:16:28,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T03:16:28,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, ASSIGN in 339 msec 2024-11-17T03:16:28,503 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T03:16:28,503 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813388503"}]},"ts":"1731813388503"} 2024-11-17T03:16:28,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T03:16:28,505 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T03:16:28,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-17T03:16:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T03:16:29,487 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:29,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe8d753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,489 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409b9eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,490 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a2b8da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,492 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:16:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:16:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20d6e7e6 2024-11-17T03:16:29,493 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:16:29,494 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:16:29,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dee51f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,496 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9e16d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,497 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c21aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d395438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463a69d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,500 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4682a7ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d6985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,503 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26666322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,504 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e412a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490627bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,505 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f06ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,506 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b6653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e16e9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f16940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,509 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4941450e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794001fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,510 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1398900b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c044e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3c4fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,512 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4178ded6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,513 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1610590e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b24ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,515 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0ba117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8574dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaa8538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6089040e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,519 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4459659b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67669ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:29,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:29,521 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e207fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:29,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102e7f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:29,523 DEBUG [Thread-1898 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,523 DEBUG [Thread-1889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,523 DEBUG [Thread-1885 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,523 DEBUG [Thread-1900 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,523 DEBUG [Thread-1891 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,523 DEBUG [Thread-1894 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,524 DEBUG [Thread-1883 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-17T03:16:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,528 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,530 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:29,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:29,530 DEBUG [Thread-1887 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,530 DEBUG [Thread-1896 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:29,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:29,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,537 DEBUG [Thread-1902 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:29,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2] 2024-11-17T03:16:29,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/d20c425c12f64463b09ce38c1da8ade0 is 50, key is test_row_0/A:col10/1731813389534/Put/seqid=0 2024-11-17T03:16:29,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742242_1418 (size=14341) 2024-11-17T03:16:29,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813449566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813449570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813389559, max timestamp=1731813389573Num uniques 859; 2024-11-17T03:16:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813389559, max timestamp=1731813389573Num uniques 859; 2024-11-17T03:16:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:16:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:16:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813389559, max timestamp=1731813389573Num uniques 859; 2024-11-17T03:16:29,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813449613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813449615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813449616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:29,639 WARN [regionserver/695c90f4bb18:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-17T03:16:29,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:29,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813449687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813449687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813449727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813449727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813449727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:29,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:29,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:29,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:29,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:29,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813449897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813449897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813449937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813449937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813449937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:29,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/d20c425c12f64463b09ce38c1da8ade0 2024-11-17T03:16:29,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f1fd8a210a004a008252a0cf570e6b9f is 50, key is test_row_0/B:col10/1731813389534/Put/seqid=0 2024-11-17T03:16:29,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742243_1419 (size=12001) 2024-11-17T03:16:29,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:29,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:29,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:29,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:30,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:30,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813450207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813450207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813450247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813450247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813450247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:30,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:30,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f1fd8a210a004a008252a0cf570e6b9f 2024-11-17T03:16:30,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/ef75f04e57934e058c31fc04d29ac9a6 is 50, key is test_row_0/C:col10/1731813389534/Put/seqid=0 2024-11-17T03:16:30,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742244_1420 (size=12001) 2024-11-17T03:16:30,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:30,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:30,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813450727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813450727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:30,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813450757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813450767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813450767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:30,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/ef75f04e57934e058c31fc04d29ac9a6 2024-11-17T03:16:30,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/d20c425c12f64463b09ce38c1da8ade0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/d20c425c12f64463b09ce38c1da8ade0 2024-11-17T03:16:30,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/d20c425c12f64463b09ce38c1da8ade0, entries=200, sequenceid=12, filesize=14.0 K 2024-11-17T03:16:30,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f1fd8a210a004a008252a0cf570e6b9f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f1fd8a210a004a008252a0cf570e6b9f 2024-11-17T03:16:30,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f1fd8a210a004a008252a0cf570e6b9f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T03:16:30,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/ef75f04e57934e058c31fc04d29ac9a6 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ef75f04e57934e058c31fc04d29ac9a6 2024-11-17T03:16:30,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ef75f04e57934e058c31fc04d29ac9a6, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T03:16:30,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1293ms, sequenceid=12, compaction requested=false 2024-11-17T03:16:30,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:30,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:30,913 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:30,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/2c372f58b8fe466cac8b14c65dc90936 is 50, key is test_row_0/A:col10/1731813389614/Put/seqid=0 2024-11-17T03:16:30,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742245_1421 (size=12001) 2024-11-17T03:16:31,323 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/2c372f58b8fe466cac8b14c65dc90936 2024-11-17T03:16:31,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c369b8f2294d455d8318d3f9fd2b0d00 is 50, key is test_row_0/B:col10/1731813389614/Put/seqid=0 2024-11-17T03:16:31,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742246_1422 (size=12001) 2024-11-17T03:16:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:31,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813451766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813451767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,778 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:31,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813451779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:31,782 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c369b8f2294d455d8318d3f9fd2b0d00 2024-11-17T03:16:31,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6625eb373a364b529b431aaf740983a0 is 50, key is test_row_0/C:col10/1731813389614/Put/seqid=0 2024-11-17T03:16:31,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813451787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813451788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,794 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,795 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:31,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742247_1423 (size=12001) 2024-11-17T03:16:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813451886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813451886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813452097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813452106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,198 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6625eb373a364b529b431aaf740983a0 2024-11-17T03:16:32,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/2c372f58b8fe466cac8b14c65dc90936 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/2c372f58b8fe466cac8b14c65dc90936 2024-11-17T03:16:32,206 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/2c372f58b8fe466cac8b14c65dc90936, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T03:16:32,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c369b8f2294d455d8318d3f9fd2b0d00 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c369b8f2294d455d8318d3f9fd2b0d00 2024-11-17T03:16:32,210 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c369b8f2294d455d8318d3f9fd2b0d00, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T03:16:32,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6625eb373a364b529b431aaf740983a0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6625eb373a364b529b431aaf740983a0 2024-11-17T03:16:32,213 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6625eb373a364b529b431aaf740983a0, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T03:16:32,214 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1301ms, sequenceid=46, compaction requested=false 2024-11-17T03:16:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:32,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-17T03:16:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-17T03:16:32,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-17T03:16:32,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6850 sec 2024-11-17T03:16:32,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.6930 sec 2024-11-17T03:16:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:32,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:32,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/c01db0aaf176411c8289cd1bc7c451fc is 50, key is test_row_0/A:col10/1731813391775/Put/seqid=0 2024-11-17T03:16:32,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742248_1424 (size=14341) 2024-11-17T03:16:32,477 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T03:16:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813452476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813452482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813452597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813452597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813452807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813452807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:32,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/c01db0aaf176411c8289cd1bc7c451fc 2024-11-17T03:16:32,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b386a41d89b04428af6fb5545ebd363e is 50, key is test_row_0/B:col10/1731813391775/Put/seqid=0 2024-11-17T03:16:32,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742249_1425 (size=12001) 2024-11-17T03:16:33,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813453116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813453116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b386a41d89b04428af6fb5545ebd363e 2024-11-17T03:16:33,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/be577d434914424c811d9cbd7dc55233 is 50, key is test_row_0/C:col10/1731813391775/Put/seqid=0 2024-11-17T03:16:33,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742250_1426 (size=12001) 2024-11-17T03:16:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813453637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813453637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/be577d434914424c811d9cbd7dc55233 2024-11-17T03:16:33,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/c01db0aaf176411c8289cd1bc7c451fc as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c01db0aaf176411c8289cd1bc7c451fc 2024-11-17T03:16:33,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c01db0aaf176411c8289cd1bc7c451fc, entries=200, sequenceid=58, filesize=14.0 K 2024-11-17T03:16:33,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b386a41d89b04428af6fb5545ebd363e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b386a41d89b04428af6fb5545ebd363e 2024-11-17T03:16:33,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b386a41d89b04428af6fb5545ebd363e, entries=150, sequenceid=58, filesize=11.7 K 2024-11-17T03:16:33,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/be577d434914424c811d9cbd7dc55233 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/be577d434914424c811d9cbd7dc55233 2024-11-17T03:16:33,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/be577d434914424c811d9cbd7dc55233, entries=150, sequenceid=58, filesize=11.7 K 2024-11-17T03:16:33,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1242ms, sequenceid=58, compaction requested=true 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:33,663 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:33,663 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:33,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:33,664 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:33,664 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:33,664 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:33,664 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:33,664 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:33,664 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:33,664 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/d20c425c12f64463b09ce38c1da8ade0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/2c372f58b8fe466cac8b14c65dc90936, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c01db0aaf176411c8289cd1bc7c451fc] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=39.7 K 2024-11-17T03:16:33,664 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f1fd8a210a004a008252a0cf570e6b9f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c369b8f2294d455d8318d3f9fd2b0d00, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b386a41d89b04428af6fb5545ebd363e] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=35.2 K 2024-11-17T03:16:33,664 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fd8a210a004a008252a0cf570e6b9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731813389533 2024-11-17T03:16:33,665 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20c425c12f64463b09ce38c1da8ade0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731813389532 2024-11-17T03:16:33,665 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c369b8f2294d455d8318d3f9fd2b0d00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731813389612 2024-11-17T03:16:33,665 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c372f58b8fe466cac8b14c65dc90936, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731813389612 2024-11-17T03:16:33,665 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b386a41d89b04428af6fb5545ebd363e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731813391752 2024-11-17T03:16:33,665 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01db0aaf176411c8289cd1bc7c451fc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731813391752 2024-11-17T03:16:33,672 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:33,672 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/cca1be76e8c74665bba89bbb0f2babfa is 50, key is test_row_0/B:col10/1731813391775/Put/seqid=0 2024-11-17T03:16:33,674 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:33,675 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/6cbcf3b7d84148c6bc687bd4ae8a7214 is 50, key is test_row_0/A:col10/1731813391775/Put/seqid=0 2024-11-17T03:16:33,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742251_1427 (size=12104) 2024-11-17T03:16:33,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742252_1428 (size=12104) 2024-11-17T03:16:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T03:16:33,707 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:33,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-17T03:16:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:33,709 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:33,710 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:33,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:33,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T03:16:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:33,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:33,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/647830beabad486fa8662770ec812076 is 50, key is test_row_0/A:col10/1731813392469/Put/seqid=0 2024-11-17T03:16:33,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742253_1429 (size=14341) 2024-11-17T03:16:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813453827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813453828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813453829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:33,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:33,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813453947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813453947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813453947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:33,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:33,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:34,086 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/cca1be76e8c74665bba89bbb0f2babfa as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/cca1be76e8c74665bba89bbb0f2babfa 2024-11-17T03:16:34,090 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into cca1be76e8c74665bba89bbb0f2babfa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:34,090 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:34,090 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/6cbcf3b7d84148c6bc687bd4ae8a7214 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/6cbcf3b7d84148c6bc687bd4ae8a7214 2024-11-17T03:16:34,090 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=13, startTime=1731813393663; duration=0sec 2024-11-17T03:16:34,090 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:34,090 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:34,090 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:34,091 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:34,091 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:34,091 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,091 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ef75f04e57934e058c31fc04d29ac9a6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6625eb373a364b529b431aaf740983a0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/be577d434914424c811d9cbd7dc55233] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=35.2 K 2024-11-17T03:16:34,091 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ef75f04e57934e058c31fc04d29ac9a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731813389533 2024-11-17T03:16:34,092 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6625eb373a364b529b431aaf740983a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731813389612 2024-11-17T03:16:34,092 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting be577d434914424c811d9cbd7dc55233, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731813391752 2024-11-17T03:16:34,094 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into 6cbcf3b7d84148c6bc687bd4ae8a7214(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:34,094 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:34,094 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=13, startTime=1731813393663; duration=0sec 2024-11-17T03:16:34,094 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:34,094 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:34,101 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:34,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/3cfe777c7fbe48778649ec5eed7935ff is 50, key is test_row_0/C:col10/1731813391775/Put/seqid=0 2024-11-17T03:16:34,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742254_1430 (size=12104) 2024-11-17T03:16:34,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813454167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813454167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813454167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/647830beabad486fa8662770ec812076 2024-11-17T03:16:34,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/0e579893441f4f4aabf999531b5ee2cf is 50, key is test_row_0/B:col10/1731813392469/Put/seqid=0 2024-11-17T03:16:34,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742255_1431 (size=12001) 2024-11-17T03:16:34,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:34,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813454476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813454476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813454477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,508 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/3cfe777c7fbe48778649ec5eed7935ff as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/3cfe777c7fbe48778649ec5eed7935ff 2024-11-17T03:16:34,511 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into 3cfe777c7fbe48778649ec5eed7935ff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:34,511 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:34,511 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=13, startTime=1731813393663; duration=0sec 2024-11-17T03:16:34,511 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:34,511 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:34,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/0e579893441f4f4aabf999531b5ee2cf 2024-11-17T03:16:34,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/69d9b03d02a047b5a66e9ffa506633f3 is 50, key is test_row_0/C:col10/1731813392469/Put/seqid=0 2024-11-17T03:16:34,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742256_1432 (size=12001) 2024-11-17T03:16:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813454657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813454657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,662 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,662 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:34,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:34,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:34,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:34,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813454987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813454987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:34,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813454987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:35,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/69d9b03d02a047b5a66e9ffa506633f3 2024-11-17T03:16:35,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/647830beabad486fa8662770ec812076 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/647830beabad486fa8662770ec812076 2024-11-17T03:16:35,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/647830beabad486fa8662770ec812076, entries=200, sequenceid=79, filesize=14.0 K 2024-11-17T03:16:35,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/0e579893441f4f4aabf999531b5ee2cf as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/0e579893441f4f4aabf999531b5ee2cf 2024-11-17T03:16:35,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/0e579893441f4f4aabf999531b5ee2cf, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T03:16:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/69d9b03d02a047b5a66e9ffa506633f3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/69d9b03d02a047b5a66e9ffa506633f3 2024-11-17T03:16:35,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/69d9b03d02a047b5a66e9ffa506633f3, entries=150, sequenceid=79, filesize=11.7 K 2024-11-17T03:16:35,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1242ms, sequenceid=79, compaction requested=false 2024-11-17T03:16:35,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:35,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T03:16:35,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:35,082 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:16:35,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:35,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/fb9ea70b1e074c68915e5fe2390ad500 is 50, key is test_row_0/A:col10/1731813393828/Put/seqid=0 2024-11-17T03:16:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742257_1433 (size=12001) 2024-11-17T03:16:35,490 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/fb9ea70b1e074c68915e5fe2390ad500 2024-11-17T03:16:35,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b37769c5c4c44cda971440fa722dbeeb is 50, key is test_row_0/B:col10/1731813393828/Put/seqid=0 2024-11-17T03:16:35,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742258_1434 (size=12001) 2024-11-17T03:16:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:35,899 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b37769c5c4c44cda971440fa722dbeeb 2024-11-17T03:16:35,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/4462cad6956d4dcf87f902c279e9bb80 is 50, key is test_row_0/C:col10/1731813393828/Put/seqid=0 2024-11-17T03:16:35,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742259_1435 (size=12001) 2024-11-17T03:16:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:36,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:36,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813456042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813396028, max timestamp=1731813396045Num uniques 859; 2024-11-17T03:16:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:16:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813396028, max timestamp=1731813396045Num uniques 859; 2024-11-17T03:16:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731813396028, max timestamp=1731813396045Num uniques 859; 2024-11-17T03:16:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:16:36,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T03:16:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813456097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813456105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813456157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813456216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813456216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,310 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/4462cad6956d4dcf87f902c279e9bb80 2024-11-17T03:16:36,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/fb9ea70b1e074c68915e5fe2390ad500 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fb9ea70b1e074c68915e5fe2390ad500 2024-11-17T03:16:36,316 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fb9ea70b1e074c68915e5fe2390ad500, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T03:16:36,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b37769c5c4c44cda971440fa722dbeeb as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b37769c5c4c44cda971440fa722dbeeb 2024-11-17T03:16:36,320 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b37769c5c4c44cda971440fa722dbeeb, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T03:16:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/4462cad6956d4dcf87f902c279e9bb80 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/4462cad6956d4dcf87f902c279e9bb80 2024-11-17T03:16:36,323 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/4462cad6956d4dcf87f902c279e9bb80, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T03:16:36,369 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1287ms, sequenceid=93, compaction requested=true 2024-11-17T03:16:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:36,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-17T03:16:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-17T03:16:36,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T03:16:36,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6600 sec 2024-11-17T03:16:36,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6650 sec 2024-11-17T03:16:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:36,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T03:16:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:36,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:36,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/fac953eec50b4c93b3bd43d4ea559843 is 50, key is test_row_0/A:col10/1731813396374/Put/seqid=0 2024-11-17T03:16:36,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742260_1436 (size=14391) 2024-11-17T03:16:36,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813456437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813456437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813456438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813456556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813456686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813456687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,700 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,701 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813456757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813456757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813456767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:36,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/fac953eec50b4c93b3bd43d4ea559843 2024-11-17T03:16:36,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/2908202520e743f28564ac5a34363b43 is 50, key is test_row_0/B:col10/1731813396374/Put/seqid=0 2024-11-17T03:16:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742261_1437 (size=12051) 2024-11-17T03:16:37,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813457077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:37,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/2908202520e743f28564ac5a34363b43 2024-11-17T03:16:37,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/f7256558db9e4520a847cda214e8dddd is 50, key is test_row_0/C:col10/1731813396374/Put/seqid=0 2024-11-17T03:16:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742262_1438 (size=12051) 2024-11-17T03:16:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813457276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:37,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813457286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:37,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/f7256558db9e4520a847cda214e8dddd 2024-11-17T03:16:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813457597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:37,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/fac953eec50b4c93b3bd43d4ea559843 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fac953eec50b4c93b3bd43d4ea559843 2024-11-17T03:16:37,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fac953eec50b4c93b3bd43d4ea559843, entries=200, sequenceid=129, filesize=14.1 K 2024-11-17T03:16:37,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/2908202520e743f28564ac5a34363b43 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2908202520e743f28564ac5a34363b43 2024-11-17T03:16:37,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2908202520e743f28564ac5a34363b43, entries=150, sequenceid=129, filesize=11.8 K 2024-11-17T03:16:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/f7256558db9e4520a847cda214e8dddd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/f7256558db9e4520a847cda214e8dddd 2024-11-17T03:16:37,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/f7256558db9e4520a847cda214e8dddd, entries=150, sequenceid=129, filesize=11.8 K 2024-11-17T03:16:37,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1242ms, sequenceid=129, compaction requested=true 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:37,617 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:37,617 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:37,618 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:37,618 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52837 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:37,618 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:37,618 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:37,618 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:37,618 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:37,619 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/cca1be76e8c74665bba89bbb0f2babfa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/0e579893441f4f4aabf999531b5ee2cf, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b37769c5c4c44cda971440fa722dbeeb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2908202520e743f28564ac5a34363b43] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=47.0 K 2024-11-17T03:16:37,619 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/6cbcf3b7d84148c6bc687bd4ae8a7214, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/647830beabad486fa8662770ec812076, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fb9ea70b1e074c68915e5fe2390ad500, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fac953eec50b4c93b3bd43d4ea559843] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=51.6 K 2024-11-17T03:16:37,619 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting cca1be76e8c74665bba89bbb0f2babfa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731813391752 2024-11-17T03:16:37,619 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e579893441f4f4aabf999531b5ee2cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731813392464 2024-11-17T03:16:37,619 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cbcf3b7d84148c6bc687bd4ae8a7214, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731813391752 2024-11-17T03:16:37,619 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b37769c5c4c44cda971440fa722dbeeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731813393813 2024-11-17T03:16:37,619 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 647830beabad486fa8662770ec812076, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731813392464 2024-11-17T03:16:37,620 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9ea70b1e074c68915e5fe2390ad500, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731813393813 2024-11-17T03:16:37,620 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2908202520e743f28564ac5a34363b43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813396084 2024-11-17T03:16:37,620 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac953eec50b4c93b3bd43d4ea559843, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813396084 2024-11-17T03:16:37,626 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:37,627 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/57285d957a5143ab9b04217d5f2aedf9 is 50, key is test_row_0/A:col10/1731813396374/Put/seqid=0 2024-11-17T03:16:37,627 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:37,628 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7b3ba0395e5648e1b5637abb6837ba57 is 50, key is test_row_0/B:col10/1731813396374/Put/seqid=0 2024-11-17T03:16:37,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742263_1439 (size=12291) 2024-11-17T03:16:37,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742264_1440 (size=12291) 2024-11-17T03:16:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T03:16:37,867 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:37,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T03:16:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T03:16:37,869 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:37,869 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:37,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T03:16:38,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T03:16:38,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:38,022 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:16:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:38,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:38,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/41d4b7e948164301a1b721b3fdbde887 is 50, key is test_row_0/A:col10/1731813396437/Put/seqid=0 2024-11-17T03:16:38,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742265_1441 (size=12151) 2024-11-17T03:16:38,044 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/57285d957a5143ab9b04217d5f2aedf9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/57285d957a5143ab9b04217d5f2aedf9 2024-11-17T03:16:38,045 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7b3ba0395e5648e1b5637abb6837ba57 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7b3ba0395e5648e1b5637abb6837ba57 2024-11-17T03:16:38,048 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into 57285d957a5143ab9b04217d5f2aedf9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:38,048 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:38,048 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=12, startTime=1731813397617; duration=0sec 2024-11-17T03:16:38,048 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:38,048 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:38,048 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:38,049 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into 7b3ba0395e5648e1b5637abb6837ba57(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:38,049 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:38,049 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=12, startTime=1731813397617; duration=0sec 2024-11-17T03:16:38,049 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:38,049 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:38,050 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:38,050 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:38,050 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:38,050 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/3cfe777c7fbe48778649ec5eed7935ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/69d9b03d02a047b5a66e9ffa506633f3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/4462cad6956d4dcf87f902c279e9bb80, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/f7256558db9e4520a847cda214e8dddd] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=47.0 K 2024-11-17T03:16:38,050 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfe777c7fbe48778649ec5eed7935ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731813391752 2024-11-17T03:16:38,051 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d9b03d02a047b5a66e9ffa506633f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731813392464 2024-11-17T03:16:38,051 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4462cad6956d4dcf87f902c279e9bb80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731813393813 2024-11-17T03:16:38,051 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7256558db9e4520a847cda214e8dddd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813396084 2024-11-17T03:16:38,058 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#369 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:38,058 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c5c37f1778de4ed7b4d4cb056a29bf0d is 50, key is test_row_0/C:col10/1731813396374/Put/seqid=0 2024-11-17T03:16:38,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742266_1442 (size=12291) 2024-11-17T03:16:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T03:16:38,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:38,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813398357, max timestamp=1731813398407Num uniques 859; 2024-11-17T03:16:38,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:16:38,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813398357, max timestamp=1731813398407Num uniques 859; 2024-11-17T03:16:38,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731813398357, max timestamp=1731813398407Num uniques 859; 2024-11-17T03:16:38,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:16:38,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T03:16:38,429 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/41d4b7e948164301a1b721b3fdbde887 2024-11-17T03:16:38,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f054dda7ddc849098ba835b60874814b is 50, key is test_row_0/B:col10/1731813396437/Put/seqid=0 2024-11-17T03:16:38,476 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c5c37f1778de4ed7b4d4cb056a29bf0d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c5c37f1778de4ed7b4d4cb056a29bf0d 2024-11-17T03:16:38,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742267_1443 (size=12151) 2024-11-17T03:16:38,486 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f054dda7ddc849098ba835b60874814b 2024-11-17T03:16:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T03:16:38,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6dad1b2432f74687b1ace79476abffc8 is 50, key is test_row_0/C:col10/1731813396437/Put/seqid=0 2024-11-17T03:16:38,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742268_1444 (size=12151) 2024-11-17T03:16:38,508 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6dad1b2432f74687b1ace79476abffc8 2024-11-17T03:16:38,510 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into c5c37f1778de4ed7b4d4cb056a29bf0d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:38,510 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:38,510 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=12, startTime=1731813397617; duration=0sec 2024-11-17T03:16:38,510 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:38,510 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/41d4b7e948164301a1b721b3fdbde887 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/41d4b7e948164301a1b721b3fdbde887 2024-11-17T03:16:38,523 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/41d4b7e948164301a1b721b3fdbde887, entries=150, sequenceid=139, filesize=11.9 K 2024-11-17T03:16:38,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f054dda7ddc849098ba835b60874814b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f054dda7ddc849098ba835b60874814b 2024-11-17T03:16:38,530 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f054dda7ddc849098ba835b60874814b, entries=150, sequenceid=139, filesize=11.9 K 2024-11-17T03:16:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6dad1b2432f74687b1ace79476abffc8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6dad1b2432f74687b1ace79476abffc8 2024-11-17T03:16:38,541 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6dad1b2432f74687b1ace79476abffc8, entries=150, sequenceid=139, filesize=11.9 K 2024-11-17T03:16:38,542 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for ed49cc07eb87ee1fb278a842d5aee3f6 in 521ms, sequenceid=139, compaction requested=false 2024-11-17T03:16:38,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:38,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:38,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T03:16:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T03:16:38,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=107.34 KB heapSize=305.30 KB 2024-11-17T03:16:38,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:38,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:38,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T03:16:38,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 677 msec 2024-11-17T03:16:38,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 684 msec 2024-11-17T03:16:38,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/27eaa45eb514471b9e8e2dc1fe317531 is 50, key is test_row_0/A:col10/1731813398412/Put/seqid=0 2024-11-17T03:16:38,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742269_1445 (size=14541) 2024-11-17T03:16:38,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/27eaa45eb514471b9e8e2dc1fe317531 2024-11-17T03:16:38,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c804e92becee4331950c8f96ad3cd652 is 50, key is test_row_0/B:col10/1731813398412/Put/seqid=0 2024-11-17T03:16:38,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742270_1446 (size=12151) 2024-11-17T03:16:38,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c804e92becee4331950c8f96ad3cd652 2024-11-17T03:16:38,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/ce7b62bc6b54485785fa8497b8ce2002 is 50, key is test_row_0/C:col10/1731813398412/Put/seqid=0 2024-11-17T03:16:38,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742271_1447 (size=12151) 2024-11-17T03:16:38,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/ce7b62bc6b54485785fa8497b8ce2002 2024-11-17T03:16:38,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/27eaa45eb514471b9e8e2dc1fe317531 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/27eaa45eb514471b9e8e2dc1fe317531 2024-11-17T03:16:38,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/27eaa45eb514471b9e8e2dc1fe317531, entries=200, sequenceid=175, filesize=14.2 K 2024-11-17T03:16:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c804e92becee4331950c8f96ad3cd652 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c804e92becee4331950c8f96ad3cd652 2024-11-17T03:16:38,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813458622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:38,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c804e92becee4331950c8f96ad3cd652, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T03:16:38,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/ce7b62bc6b54485785fa8497b8ce2002 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ce7b62bc6b54485785fa8497b8ce2002 2024-11-17T03:16:38,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ce7b62bc6b54485785fa8497b8ce2002, entries=150, sequenceid=175, filesize=11.9 K 2024-11-17T03:16:38,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~305.25 KB/312576, currentSize=73.80 KB/75570 for ed49cc07eb87ee1fb278a842d5aee3f6 in 90ms, sequenceid=175, compaction requested=true 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:38,637 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:38,637 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:38,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:38,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:16:38,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:38,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:38,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:38,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:38,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:38,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:38,640 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:38,640 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38983 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:38,640 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:38,640 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:38,640 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:38,641 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:38,641 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7b3ba0395e5648e1b5637abb6837ba57, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f054dda7ddc849098ba835b60874814b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c804e92becee4331950c8f96ad3cd652] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=35.7 K 2024-11-17T03:16:38,641 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/57285d957a5143ab9b04217d5f2aedf9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/41d4b7e948164301a1b721b3fdbde887, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/27eaa45eb514471b9e8e2dc1fe317531] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=38.1 K 2024-11-17T03:16:38,641 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3ba0395e5648e1b5637abb6837ba57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813396084 2024-11-17T03:16:38,641 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57285d957a5143ab9b04217d5f2aedf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813396084 2024-11-17T03:16:38,641 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting f054dda7ddc849098ba835b60874814b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731813396406 2024-11-17T03:16:38,641 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d4b7e948164301a1b721b3fdbde887, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731813396406 2024-11-17T03:16:38,642 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c804e92becee4331950c8f96ad3cd652, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731813398412 2024-11-17T03:16:38,642 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27eaa45eb514471b9e8e2dc1fe317531, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731813398412 2024-11-17T03:16:38,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/be46587446464806b9f46a292dcc4230 is 50, key is test_row_0/A:col10/1731813398597/Put/seqid=0 2024-11-17T03:16:38,659 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:38,660 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/c6a0d47cec654c82b4b5e3f0c6b5a371 is 50, key is test_row_0/A:col10/1731813398412/Put/seqid=0 2024-11-17T03:16:38,661 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:38,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742272_1448 (size=16931) 2024-11-17T03:16:38,661 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/2c872d6728eb435e9d66667b3f230bbd is 50, key is test_row_0/B:col10/1731813398412/Put/seqid=0 2024-11-17T03:16:38,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/be46587446464806b9f46a292dcc4230 2024-11-17T03:16:38,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742274_1450 (size=12493) 2024-11-17T03:16:38,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7fa3b5384b0d4378b67df15b7c68c7fe is 50, key is test_row_0/B:col10/1731813398597/Put/seqid=0 2024-11-17T03:16:38,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742273_1449 (size=12493) 2024-11-17T03:16:38,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742275_1451 (size=12151) 2024-11-17T03:16:38,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7fa3b5384b0d4378b67df15b7c68c7fe 2024-11-17T03:16:38,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/b516f2cf29834bb5bd822134c9af25a1 is 50, key is test_row_0/C:col10/1731813398597/Put/seqid=0 2024-11-17T03:16:38,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742276_1452 (size=12151) 2024-11-17T03:16:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813458706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813458720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813458737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813458827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:38,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813458837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813458957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T03:16:39,006 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:39,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T03:16:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T03:16:39,008 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:39,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:39,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813459047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813459047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,076 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/2c872d6728eb435e9d66667b3f230bbd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2c872d6728eb435e9d66667b3f230bbd 2024-11-17T03:16:39,079 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into 2c872d6728eb435e9d66667b3f230bbd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:39,079 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:39,079 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=13, startTime=1731813398637; duration=0sec 2024-11-17T03:16:39,079 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:39,079 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:39,079 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:39,079 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:39,080 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:39,080 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:39,080 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c5c37f1778de4ed7b4d4cb056a29bf0d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6dad1b2432f74687b1ace79476abffc8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ce7b62bc6b54485785fa8497b8ce2002] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=35.7 K 2024-11-17T03:16:39,080 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c37f1778de4ed7b4d4cb056a29bf0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813396084 2024-11-17T03:16:39,080 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/c6a0d47cec654c82b4b5e3f0c6b5a371 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c6a0d47cec654c82b4b5e3f0c6b5a371 2024-11-17T03:16:39,080 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dad1b2432f74687b1ace79476abffc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731813396406 2024-11-17T03:16:39,080 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7b62bc6b54485785fa8497b8ce2002, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731813398412 2024-11-17T03:16:39,087 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:39,087 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/a2a9859dddac442192e064d6311e8c44 is 50, key is test_row_0/C:col10/1731813398412/Put/seqid=0 2024-11-17T03:16:39,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742277_1453 (size=12493) 2024-11-17T03:16:39,090 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into c6a0d47cec654c82b4b5e3f0c6b5a371(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:39,090 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:39,090 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=13, startTime=1731813398637; duration=0sec 2024-11-17T03:16:39,090 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:39,090 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:39,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/b516f2cf29834bb5bd822134c9af25a1 2024-11-17T03:16:39,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/be46587446464806b9f46a292dcc4230 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/be46587446464806b9f46a292dcc4230 2024-11-17T03:16:39,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/be46587446464806b9f46a292dcc4230, entries=250, sequenceid=191, filesize=16.5 K 2024-11-17T03:16:39,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7fa3b5384b0d4378b67df15b7c68c7fe as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa3b5384b0d4378b67df15b7c68c7fe 2024-11-17T03:16:39,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa3b5384b0d4378b67df15b7c68c7fe, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T03:16:39,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/b516f2cf29834bb5bd822134c9af25a1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/b516f2cf29834bb5bd822134c9af25a1 2024-11-17T03:16:39,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/b516f2cf29834bb5bd822134c9af25a1, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T03:16:39,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ed49cc07eb87ee1fb278a842d5aee3f6 in 474ms, sequenceid=191, compaction requested=false 2024-11-17T03:16:39,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T03:16:39,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T03:16:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:39,161 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T03:16:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:39,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:39,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/14d228e31fa343e7ad741ae455c2c419 is 50, key is test_row_0/A:col10/1731813398717/Put/seqid=0 2024-11-17T03:16:39,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742278_1454 (size=12151) 2024-11-17T03:16:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T03:16:39,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813459339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813459357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813459357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813459446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,493 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/a2a9859dddac442192e064d6311e8c44 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/a2a9859dddac442192e064d6311e8c44 2024-11-17T03:16:39,496 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into a2a9859dddac442192e064d6311e8c44(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:39,496 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:39,496 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=13, startTime=1731813398637; duration=0sec 2024-11-17T03:16:39,496 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:39,496 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:39,570 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/14d228e31fa343e7ad741ae455c2c419 2024-11-17T03:16:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/858a6a4e4cf245759a846f5ab271065e is 50, key is test_row_0/B:col10/1731813398717/Put/seqid=0 2024-11-17T03:16:39,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742279_1455 (size=12151) 2024-11-17T03:16:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T03:16:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813459667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813459877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813459878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:39,978 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/858a6a4e4cf245759a846f5ab271065e 2024-11-17T03:16:39,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/81b5649d209442249b3f7efb5bdf26c7 is 50, key is test_row_0/C:col10/1731813398717/Put/seqid=0 2024-11-17T03:16:39,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742280_1456 (size=12151) 2024-11-17T03:16:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813459986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T03:16:40,387 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/81b5649d209442249b3f7efb5bdf26c7 2024-11-17T03:16:40,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/14d228e31fa343e7ad741ae455c2c419 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/14d228e31fa343e7ad741ae455c2c419 2024-11-17T03:16:40,393 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/14d228e31fa343e7ad741ae455c2c419, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T03:16:40,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/858a6a4e4cf245759a846f5ab271065e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/858a6a4e4cf245759a846f5ab271065e 2024-11-17T03:16:40,398 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/858a6a4e4cf245759a846f5ab271065e, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T03:16:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/81b5649d209442249b3f7efb5bdf26c7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81b5649d209442249b3f7efb5bdf26c7 2024-11-17T03:16:40,401 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81b5649d209442249b3f7efb5bdf26c7, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T03:16:40,402 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1241ms, sequenceid=209, compaction requested=true 2024-11-17T03:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T03:16:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T03:16:40,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T03:16:40,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3940 sec 2024-11-17T03:16:40,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3980 sec 2024-11-17T03:16:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:40,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:16:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:40,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/e3fa182ff2e04cc8b703ed51dd3812fd is 50, key is test_row_0/A:col10/1731813400501/Put/seqid=0 2024-11-17T03:16:40,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742281_1457 (size=14541) 2024-11-17T03:16:40,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/e3fa182ff2e04cc8b703ed51dd3812fd 2024-11-17T03:16:40,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/02db5be115084678b0ae0aee9ac96627 is 50, key is test_row_0/B:col10/1731813400501/Put/seqid=0 2024-11-17T03:16:40,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742282_1458 (size=12151) 2024-11-17T03:16:40,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813460591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813460706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44664 deadline: 1731813460737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44722 deadline: 1731813460737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,745 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:40,745 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813460897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813460897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,906 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,906 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:40,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/02db5be115084678b0ae0aee9ac96627 2024-11-17T03:16:40,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/bc404b765af040f78145419e43efb2a2 is 50, key is test_row_0/C:col10/1731813400501/Put/seqid=0 2024-11-17T03:16:40,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742283_1459 (size=12151) 2024-11-17T03:16:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813460927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T03:16:41,157 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:41,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T03:16:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T03:16:41,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:41,159 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:41,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813461247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:41,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T03:16:41,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T03:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:41,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:41,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/bc404b765af040f78145419e43efb2a2 2024-11-17T03:16:41,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/e3fa182ff2e04cc8b703ed51dd3812fd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/e3fa182ff2e04cc8b703ed51dd3812fd 2024-11-17T03:16:41,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/e3fa182ff2e04cc8b703ed51dd3812fd, entries=200, sequenceid=227, filesize=14.2 K 2024-11-17T03:16:41,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/02db5be115084678b0ae0aee9ac96627 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/02db5be115084678b0ae0aee9ac96627 2024-11-17T03:16:41,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/02db5be115084678b0ae0aee9ac96627, entries=150, sequenceid=227, filesize=11.9 K 2024-11-17T03:16:41,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/bc404b765af040f78145419e43efb2a2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/bc404b765af040f78145419e43efb2a2 2024-11-17T03:16:41,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/bc404b765af040f78145419e43efb2a2, entries=150, sequenceid=227, filesize=11.9 K 2024-11-17T03:16:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ed49cc07eb87ee1fb278a842d5aee3f6 in 838ms, sequenceid=227, compaction requested=true 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:41,340 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:41,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:41,340 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:41,341 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:41,341 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:41,341 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:41,341 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:41,341 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:41,341 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:41,341 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c6a0d47cec654c82b4b5e3f0c6b5a371, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/be46587446464806b9f46a292dcc4230, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/14d228e31fa343e7ad741ae455c2c419, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/e3fa182ff2e04cc8b703ed51dd3812fd] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=54.8 K 2024-11-17T03:16:41,341 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2c872d6728eb435e9d66667b3f230bbd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa3b5384b0d4378b67df15b7c68c7fe, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/858a6a4e4cf245759a846f5ab271065e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/02db5be115084678b0ae0aee9ac96627] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=47.8 K 2024-11-17T03:16:41,341 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a0d47cec654c82b4b5e3f0c6b5a371, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731813398412 2024-11-17T03:16:41,341 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c872d6728eb435e9d66667b3f230bbd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731813398412 2024-11-17T03:16:41,342 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting be46587446464806b9f46a292dcc4230, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731813398597 2024-11-17T03:16:41,342 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa3b5384b0d4378b67df15b7c68c7fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731813398597 2024-11-17T03:16:41,342 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d228e31fa343e7ad741ae455c2c419, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731813398656 2024-11-17T03:16:41,342 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 858a6a4e4cf245759a846f5ab271065e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731813398656 2024-11-17T03:16:41,342 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3fa182ff2e04cc8b703ed51dd3812fd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731813399307 2024-11-17T03:16:41,342 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 02db5be115084678b0ae0aee9ac96627, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731813399307 2024-11-17T03:16:41,347 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:41,347 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/cc258e8524cf4efab0620a21f13a7402 is 50, key is test_row_0/A:col10/1731813400501/Put/seqid=0 2024-11-17T03:16:41,348 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:41,348 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/8a24bbf63d4644b5a824681cdebc564e is 50, key is test_row_0/B:col10/1731813400501/Put/seqid=0 2024-11-17T03:16:41,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742285_1461 (size=12629) 2024-11-17T03:16:41,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742284_1460 (size=12629) 2024-11-17T03:16:41,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T03:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:41,463 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:16:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:41,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/3d11991565d443aaac0acfd3eabcfe3b is 50, key is test_row_0/A:col10/1731813400549/Put/seqid=0 2024-11-17T03:16:41,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742286_1462 (size=12151) 2024-11-17T03:16:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T03:16:41,755 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/8a24bbf63d4644b5a824681cdebc564e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a24bbf63d4644b5a824681cdebc564e 2024-11-17T03:16:41,755 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/cc258e8524cf4efab0620a21f13a7402 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cc258e8524cf4efab0620a21f13a7402 2024-11-17T03:16:41,759 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into cc258e8524cf4efab0620a21f13a7402(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:41,759 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:41,759 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=12, startTime=1731813401340; duration=0sec 2024-11-17T03:16:41,759 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:41,759 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:41,759 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:16:41,761 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into 8a24bbf63d4644b5a824681cdebc564e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:41,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:41,761 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=12, startTime=1731813401340; duration=0sec 2024-11-17T03:16:41,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:41,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:41,763 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:16:41,763 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:41,763 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:41,763 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/a2a9859dddac442192e064d6311e8c44, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/b516f2cf29834bb5bd822134c9af25a1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81b5649d209442249b3f7efb5bdf26c7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/bc404b765af040f78145419e43efb2a2] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=47.8 K 2024-11-17T03:16:41,763 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a9859dddac442192e064d6311e8c44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731813398412 2024-11-17T03:16:41,764 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b516f2cf29834bb5bd822134c9af25a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731813398597 2024-11-17T03:16:41,764 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b5649d209442249b3f7efb5bdf26c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731813398656 2024-11-17T03:16:41,764 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc404b765af040f78145419e43efb2a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731813399307 2024-11-17T03:16:41,771 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:41,771 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/81ec0d79091b47b7887e35feb8b10754 is 50, key is test_row_0/C:col10/1731813400501/Put/seqid=0 2024-11-17T03:16:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:41,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:41,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742287_1463 (size=12629) 2024-11-17T03:16:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T03:16:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813461847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:41,873 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/3d11991565d443aaac0acfd3eabcfe3b 2024-11-17T03:16:41,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/58108cb3412f43879fed2a7e8b246a7c is 50, key is test_row_0/B:col10/1731813400549/Put/seqid=0 2024-11-17T03:16:41,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742288_1464 (size=12151) 2024-11-17T03:16:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813461957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813462167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:42,192 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/81ec0d79091b47b7887e35feb8b10754 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81ec0d79091b47b7887e35feb8b10754 2024-11-17T03:16:42,198 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into 81ec0d79091b47b7887e35feb8b10754(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:42,198 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:42,198 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=12, startTime=1731813401340; duration=0sec 2024-11-17T03:16:42,198 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:42,198 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:42,282 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/58108cb3412f43879fed2a7e8b246a7c 2024-11-17T03:16:42,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/9146c5d6d3bb446fa282dccdeb7983ff is 50, key is test_row_0/C:col10/1731813400549/Put/seqid=0 2024-11-17T03:16:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742289_1465 (size=12151) 2024-11-17T03:16:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T03:16:42,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813462487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:42,694 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/9146c5d6d3bb446fa282dccdeb7983ff 2024-11-17T03:16:42,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/3d11991565d443aaac0acfd3eabcfe3b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/3d11991565d443aaac0acfd3eabcfe3b 2024-11-17T03:16:42,699 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/3d11991565d443aaac0acfd3eabcfe3b, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T03:16:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/58108cb3412f43879fed2a7e8b246a7c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/58108cb3412f43879fed2a7e8b246a7c 2024-11-17T03:16:42,702 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/58108cb3412f43879fed2a7e8b246a7c, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T03:16:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/9146c5d6d3bb446fa282dccdeb7983ff as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/9146c5d6d3bb446fa282dccdeb7983ff 2024-11-17T03:16:42,705 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/9146c5d6d3bb446fa282dccdeb7983ff, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T03:16:42,706 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1244ms, sequenceid=242, compaction requested=false 2024-11-17T03:16:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T03:16:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T03:16:42,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T03:16:42,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5490 sec 2024-11-17T03:16:42,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5520 sec 2024-11-17T03:16:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:42,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:16:42,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:42,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:42,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:42,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:42,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:42,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:42,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/1c8f1400c5e64bb78f9d1675273a5b92 is 50, key is test_row_0/A:col10/1731813401834/Put/seqid=0 2024-11-17T03:16:42,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742290_1466 (size=14741) 2024-11-17T03:16:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813462983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813462985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813462997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813463097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813463097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T03:16:43,307 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:43,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T03:16:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T03:16:43,308 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:43,309 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:43,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:43,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813463307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813463307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/1c8f1400c5e64bb78f9d1675273a5b92 2024-11-17T03:16:43,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f8914cfd236c448c9e1d6561145b59e2 is 50, key is test_row_0/B:col10/1731813401834/Put/seqid=0 2024-11-17T03:16:43,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742291_1467 (size=12301) 2024-11-17T03:16:43,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f8914cfd236c448c9e1d6561145b59e2 2024-11-17T03:16:43,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c47c3eac5ed443d6ad717511dfa20c65 is 50, key is test_row_0/C:col10/1731813401834/Put/seqid=0 2024-11-17T03:16:43,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742292_1468 (size=12301) 2024-11-17T03:16:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T03:16:43,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T03:16:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:43,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:43,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T03:16:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:43,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813463617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T03:16:43,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813463626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:43,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c47c3eac5ed443d6ad717511dfa20c65 2024-11-17T03:16:43,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/1c8f1400c5e64bb78f9d1675273a5b92 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1c8f1400c5e64bb78f9d1675273a5b92 2024-11-17T03:16:43,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1c8f1400c5e64bb78f9d1675273a5b92, entries=200, sequenceid=263, filesize=14.4 K 2024-11-17T03:16:43,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f8914cfd236c448c9e1d6561145b59e2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f8914cfd236c448c9e1d6561145b59e2 2024-11-17T03:16:43,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f8914cfd236c448c9e1d6561145b59e2, entries=150, sequenceid=263, filesize=12.0 K 2024-11-17T03:16:43,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c47c3eac5ed443d6ad717511dfa20c65 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c47c3eac5ed443d6ad717511dfa20c65 2024-11-17T03:16:43,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c47c3eac5ed443d6ad717511dfa20c65, entries=150, sequenceid=263, filesize=12.0 K 2024-11-17T03:16:43,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ed49cc07eb87ee1fb278a842d5aee3f6 in 838ms, sequenceid=263, compaction requested=true 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:43,760 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:43,760 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:43,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:43,761 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,761 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,761 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a24bbf63d4644b5a824681cdebc564e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/58108cb3412f43879fed2a7e8b246a7c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f8914cfd236c448c9e1d6561145b59e2] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.2 K 2024-11-17T03:16:43,761 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cc258e8524cf4efab0620a21f13a7402, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/3d11991565d443aaac0acfd3eabcfe3b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1c8f1400c5e64bb78f9d1675273a5b92] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=38.6 K 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a24bbf63d4644b5a824681cdebc564e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731813399307 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc258e8524cf4efab0620a21f13a7402, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731813399307 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 58108cb3412f43879fed2a7e8b246a7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731813400549 2024-11-17T03:16:43,761 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d11991565d443aaac0acfd3eabcfe3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731813400549 2024-11-17T03:16:43,762 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting f8914cfd236c448c9e1d6561145b59e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731813401827 2024-11-17T03:16:43,762 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8f1400c5e64bb78f9d1675273a5b92, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731813401827 2024-11-17T03:16:43,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T03:16:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:43,766 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:43,767 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:43,767 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:43,767 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/eb258ff544c44caeaa21519d89725ff5 is 50, key is test_row_0/B:col10/1731813401834/Put/seqid=0 2024-11-17T03:16:43,768 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/ca81fab0a12244d2acb63e66dfc12e7a is 50, key is test_row_0/A:col10/1731813401834/Put/seqid=0 2024-11-17T03:16:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/652a1c00c0de436f8fe87d4193031582 is 50, key is test_row_0/A:col10/1731813402980/Put/seqid=0 2024-11-17T03:16:43,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742293_1469 (size=12881) 2024-11-17T03:16:43,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742294_1470 (size=12881) 2024-11-17T03:16:43,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742295_1471 (size=12301) 2024-11-17T03:16:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T03:16:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:44,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:44,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813464122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813464136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813464136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:44,180 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/ca81fab0a12244d2acb63e66dfc12e7a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/ca81fab0a12244d2acb63e66dfc12e7a 2024-11-17T03:16:44,180 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/eb258ff544c44caeaa21519d89725ff5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/eb258ff544c44caeaa21519d89725ff5 2024-11-17T03:16:44,181 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/652a1c00c0de436f8fe87d4193031582 2024-11-17T03:16:44,183 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into ca81fab0a12244d2acb63e66dfc12e7a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:44,183 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into eb258ff544c44caeaa21519d89725ff5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:44,184 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=13, startTime=1731813403760; duration=0sec 2024-11-17T03:16:44,184 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=13, startTime=1731813403760; duration=0sec 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:44,184 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:44,184 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:44,185 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81ec0d79091b47b7887e35feb8b10754, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/9146c5d6d3bb446fa282dccdeb7983ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c47c3eac5ed443d6ad717511dfa20c65] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.2 K 2024-11-17T03:16:44,185 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ec0d79091b47b7887e35feb8b10754, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731813399307 2024-11-17T03:16:44,185 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9146c5d6d3bb446fa282dccdeb7983ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731813400549 2024-11-17T03:16:44,185 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting c47c3eac5ed443d6ad717511dfa20c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731813401827 2024-11-17T03:16:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/d744affaeee0472b951304dac966ad3d is 50, key is test_row_0/B:col10/1731813402980/Put/seqid=0 2024-11-17T03:16:44,192 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:44,192 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/e292a951eba449d7a06077c2ebad0a6c is 50, key is test_row_0/C:col10/1731813401834/Put/seqid=0 2024-11-17T03:16:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742296_1472 (size=12301) 2024-11-17T03:16:44,198 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/d744affaeee0472b951304dac966ad3d 2024-11-17T03:16:44,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/353d55a162104d49b5844af3857ba384 is 50, key is test_row_0/C:col10/1731813402980/Put/seqid=0 2024-11-17T03:16:44,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742297_1473 (size=12881) 2024-11-17T03:16:44,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742298_1474 (size=12301) 2024-11-17T03:16:44,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813464236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T03:16:44,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813464447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:44,607 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/353d55a162104d49b5844af3857ba384 2024-11-17T03:16:44,609 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/e292a951eba449d7a06077c2ebad0a6c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e292a951eba449d7a06077c2ebad0a6c 2024-11-17T03:16:44,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/652a1c00c0de436f8fe87d4193031582 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/652a1c00c0de436f8fe87d4193031582 2024-11-17T03:16:44,614 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into e292a951eba449d7a06077c2ebad0a6c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:44,614 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:44,614 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=13, startTime=1731813403760; duration=0sec 2024-11-17T03:16:44,614 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:44,614 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:44,614 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/652a1c00c0de436f8fe87d4193031582, entries=150, sequenceid=277, filesize=12.0 K 2024-11-17T03:16:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/d744affaeee0472b951304dac966ad3d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/d744affaeee0472b951304dac966ad3d 2024-11-17T03:16:44,618 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/d744affaeee0472b951304dac966ad3d, entries=150, sequenceid=277, filesize=12.0 K 2024-11-17T03:16:44,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/353d55a162104d49b5844af3857ba384 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/353d55a162104d49b5844af3857ba384 2024-11-17T03:16:44,622 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/353d55a162104d49b5844af3857ba384, entries=150, sequenceid=277, filesize=12.0 K 2024-11-17T03:16:44,623 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ed49cc07eb87ee1fb278a842d5aee3f6 in 857ms, sequenceid=277, compaction requested=false 2024-11-17T03:16:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:44,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T03:16:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T03:16:44,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T03:16:44,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-11-17T03:16:44,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3190 sec 2024-11-17T03:16:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:44,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T03:16:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:44,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/150672502d614589a038b601f8c2de16 is 50, key is test_row_0/A:col10/1731813404103/Put/seqid=0 2024-11-17T03:16:44,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742299_1475 (size=14741) 2024-11-17T03:16:44,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813464842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:44,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813464957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:45,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813465157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:45,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813465157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:45,164 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:45,164 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:45,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/150672502d614589a038b601f8c2de16 2024-11-17T03:16:45,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813465177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:45,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/391f0245643042ceb55f92d090c14bf5 is 50, key is test_row_0/B:col10/1731813404103/Put/seqid=0 2024-11-17T03:16:45,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742300_1476 (size=12301) 2024-11-17T03:16:45,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/391f0245643042ceb55f92d090c14bf5 2024-11-17T03:16:45,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/e15c14fc29354634b562176b8e352be5 is 50, key is test_row_0/C:col10/1731813404103/Put/seqid=0 2024-11-17T03:16:45,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742301_1477 (size=12301) 2024-11-17T03:16:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T03:16:45,457 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:45,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T03:16:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T03:16:45,459 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:45,460 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:45,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813465486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T03:16:45,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/e15c14fc29354634b562176b8e352be5 2024-11-17T03:16:45,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/150672502d614589a038b601f8c2de16 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/150672502d614589a038b601f8c2de16 2024-11-17T03:16:45,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/150672502d614589a038b601f8c2de16, entries=200, sequenceid=299, filesize=14.4 K 2024-11-17T03:16:45,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/391f0245643042ceb55f92d090c14bf5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/391f0245643042ceb55f92d090c14bf5 2024-11-17T03:16:45,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/391f0245643042ceb55f92d090c14bf5, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T03:16:45,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/e15c14fc29354634b562176b8e352be5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e15c14fc29354634b562176b8e352be5 2024-11-17T03:16:45,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e15c14fc29354634b562176b8e352be5, entries=150, sequenceid=299, filesize=12.0 K 2024-11-17T03:16:45,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ed49cc07eb87ee1fb278a842d5aee3f6 in 846ms, sequenceid=299, compaction requested=true 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:45,609 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:45,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T03:16:45,609 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:45,610 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:45,610 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:45,610 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:45,610 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:45,610 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:45,610 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:45,610 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/eb258ff544c44caeaa21519d89725ff5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/d744affaeee0472b951304dac966ad3d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/391f0245643042ceb55f92d090c14bf5] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.6 K 2024-11-17T03:16:45,610 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/ca81fab0a12244d2acb63e66dfc12e7a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/652a1c00c0de436f8fe87d4193031582, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/150672502d614589a038b601f8c2de16] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=39.0 K 2024-11-17T03:16:45,611 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca81fab0a12244d2acb63e66dfc12e7a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731813401827 2024-11-17T03:16:45,611 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting eb258ff544c44caeaa21519d89725ff5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731813401827 2024-11-17T03:16:45,611 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 652a1c00c0de436f8fe87d4193031582, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731813402980 2024-11-17T03:16:45,611 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting d744affaeee0472b951304dac966ad3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731813402980 2024-11-17T03:16:45,611 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 391f0245643042ceb55f92d090c14bf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731813404103 2024-11-17T03:16:45,611 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150672502d614589a038b601f8c2de16, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731813404103 2024-11-17T03:16:45,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:45,612 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/02919486bcc740a9b7cab48dc7a8f78e is 50, key is test_row_0/A:col10/1731813404826/Put/seqid=0 2024-11-17T03:16:45,617 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:45,617 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:45,618 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/f3bb649f51e04578a0be682c72f624d9 is 50, key is test_row_0/A:col10/1731813404103/Put/seqid=0 2024-11-17T03:16:45,618 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7ac3aa48767b44449d65d55ee4eb5c11 is 50, key is test_row_0/B:col10/1731813404103/Put/seqid=0 2024-11-17T03:16:45,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742303_1479 (size=12983) 2024-11-17T03:16:45,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742302_1478 (size=12301) 2024-11-17T03:16:45,623 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/02919486bcc740a9b7cab48dc7a8f78e 2024-11-17T03:16:45,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742304_1480 (size=12983) 2024-11-17T03:16:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/e1eca82ff1874c0e93cc5933d7372702 is 50, key is test_row_0/B:col10/1731813404826/Put/seqid=0 2024-11-17T03:16:45,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742305_1481 (size=12301) 2024-11-17T03:16:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T03:16:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:46,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:46,025 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/f3bb649f51e04578a0be682c72f624d9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/f3bb649f51e04578a0be682c72f624d9 2024-11-17T03:16:46,028 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7ac3aa48767b44449d65d55ee4eb5c11 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7ac3aa48767b44449d65d55ee4eb5c11 2024-11-17T03:16:46,029 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into f3bb649f51e04578a0be682c72f624d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:46,029 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:46,029 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=13, startTime=1731813405609; duration=0sec 2024-11-17T03:16:46,029 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:46,029 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:46,029 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:46,029 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:46,029 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:46,029 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:46,030 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e292a951eba449d7a06077c2ebad0a6c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/353d55a162104d49b5844af3857ba384, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e15c14fc29354634b562176b8e352be5] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.6 K 2024-11-17T03:16:46,030 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e292a951eba449d7a06077c2ebad0a6c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731813401827 2024-11-17T03:16:46,030 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353d55a162104d49b5844af3857ba384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731813402980 2024-11-17T03:16:46,030 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15c14fc29354634b562176b8e352be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731813404103 2024-11-17T03:16:46,032 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into 7ac3aa48767b44449d65d55ee4eb5c11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:46,032 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:46,032 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=13, startTime=1731813405609; duration=0sec 2024-11-17T03:16:46,032 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:46,032 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:46,033 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/e1eca82ff1874c0e93cc5933d7372702 2024-11-17T03:16:46,037 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:46,038 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6b6e486238fb470d82c8a94b0da74333 is 50, key is test_row_0/C:col10/1731813404103/Put/seqid=0 2024-11-17T03:16:46,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/52c0c487a48b4457b6e5d3905bc9f1e9 is 50, key is test_row_0/C:col10/1731813404826/Put/seqid=0 2024-11-17T03:16:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742306_1482 (size=12301) 2024-11-17T03:16:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742307_1483 (size=12983) 2024-11-17T03:16:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T03:16:46,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813466095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:46,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:46,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813466207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813466417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/52c0c487a48b4457b6e5d3905bc9f1e9 2024-11-17T03:16:46,501 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/6b6e486238fb470d82c8a94b0da74333 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6b6e486238fb470d82c8a94b0da74333 2024-11-17T03:16:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/02919486bcc740a9b7cab48dc7a8f78e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/02919486bcc740a9b7cab48dc7a8f78e 2024-11-17T03:16:46,504 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/02919486bcc740a9b7cab48dc7a8f78e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T03:16:46,504 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into 6b6e486238fb470d82c8a94b0da74333(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:46,504 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:46,504 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=13, startTime=1731813405609; duration=0sec 2024-11-17T03:16:46,504 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:46,504 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/e1eca82ff1874c0e93cc5933d7372702 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/e1eca82ff1874c0e93cc5933d7372702 2024-11-17T03:16:46,508 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/e1eca82ff1874c0e93cc5933d7372702, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T03:16:46,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/52c0c487a48b4457b6e5d3905bc9f1e9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/52c0c487a48b4457b6e5d3905bc9f1e9 2024-11-17T03:16:46,511 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/52c0c487a48b4457b6e5d3905bc9f1e9, entries=150, sequenceid=312, filesize=12.0 K 2024-11-17T03:16:46,511 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ed49cc07eb87ee1fb278a842d5aee3f6 in 899ms, sequenceid=312, compaction requested=false 2024-11-17T03:16:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:46,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:46,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T03:16:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T03:16:46,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T03:16:46,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0530 sec 2024-11-17T03:16:46,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0570 sec 2024-11-17T03:16:46,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T03:16:46,597 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:46,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T03:16:46,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:46,599 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:46,599 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:46,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:46,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T03:16:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:46,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:46,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/cd58efec8dfd4854a19bf5a61d6fc6c1 is 50, key is test_row_0/A:col10/1731813406088/Put/seqid=0 2024-11-17T03:16:46,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742308_1484 (size=14741) 2024-11-17T03:16:46,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:46,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813466789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:46,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:46,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:46,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:46,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:46,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813466896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:46,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:47,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813467107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:47,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/cd58efec8dfd4854a19bf5a61d6fc6c1 2024-11-17T03:16:47,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/8a0e1e135aef4459b4e9d35b592de6fd is 50, key is test_row_0/B:col10/1731813406088/Put/seqid=0 2024-11-17T03:16:47,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742309_1485 (size=12301) 2024-11-17T03:16:47,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/8a0e1e135aef4459b4e9d35b592de6fd 2024-11-17T03:16:47,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/67ebe4d56f684e35a175b2395cda6d3c is 50, key is test_row_0/C:col10/1731813406088/Put/seqid=0 2024-11-17T03:16:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742310_1486 (size=12301) 2024-11-17T03:16:47,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44670 deadline: 1731813467177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:47,182 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:47,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44710 deadline: 1731813467187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:47,193 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in ed49cc07eb87ee1fb278a842d5aee3f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:47,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:47,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:47,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813467427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:47,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:47,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/67ebe4d56f684e35a175b2395cda6d3c 2024-11-17T03:16:47,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/cd58efec8dfd4854a19bf5a61d6fc6c1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cd58efec8dfd4854a19bf5a61d6fc6c1 2024-11-17T03:16:47,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cd58efec8dfd4854a19bf5a61d6fc6c1, entries=200, sequenceid=335, filesize=14.4 K 2024-11-17T03:16:47,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/8a0e1e135aef4459b4e9d35b592de6fd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a0e1e135aef4459b4e9d35b592de6fd 2024-11-17T03:16:47,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a0e1e135aef4459b4e9d35b592de6fd, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T03:16:47,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/67ebe4d56f684e35a175b2395cda6d3c as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/67ebe4d56f684e35a175b2395cda6d3c 2024-11-17T03:16:47,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/67ebe4d56f684e35a175b2395cda6d3c, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T03:16:47,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ed49cc07eb87ee1fb278a842d5aee3f6 in 843ms, sequenceid=335, compaction requested=true 2024-11-17T03:16:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:47,582 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:47,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:47,582 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:47,583 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:47,583 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:47,583 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:47,583 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:47,583 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,583 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,584 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/f3bb649f51e04578a0be682c72f624d9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/02919486bcc740a9b7cab48dc7a8f78e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cd58efec8dfd4854a19bf5a61d6fc6c1] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=39.1 K 2024-11-17T03:16:47,584 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7ac3aa48767b44449d65d55ee4eb5c11, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/e1eca82ff1874c0e93cc5933d7372702, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a0e1e135aef4459b4e9d35b592de6fd] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.7 K 2024-11-17T03:16:47,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac3aa48767b44449d65d55ee4eb5c11, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731813404103 2024-11-17T03:16:47,584 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3bb649f51e04578a0be682c72f624d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731813404103 2024-11-17T03:16:47,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting e1eca82ff1874c0e93cc5933d7372702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731813404818 2024-11-17T03:16:47,584 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02919486bcc740a9b7cab48dc7a8f78e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731813404818 2024-11-17T03:16:47,584 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0e1e135aef4459b4e9d35b592de6fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731813406078 2024-11-17T03:16:47,584 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd58efec8dfd4854a19bf5a61d6fc6c1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731813406078 2024-11-17T03:16:47,590 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:47,590 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:47,590 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c3226fc21f204ecca0823703a59c9065 is 50, key is test_row_0/B:col10/1731813406088/Put/seqid=0 2024-11-17T03:16:47,590 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/197813e56d3f4cc281dc348a89b49de9 is 50, key is test_row_0/A:col10/1731813406088/Put/seqid=0 2024-11-17T03:16:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742311_1487 (size=13085) 2024-11-17T03:16:47,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742312_1488 (size=13085) 2024-11-17T03:16:47,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T03:16:47,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:47,668 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:16:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:47,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/1e77249deee24dac9995f1cfa81f9ff0 is 50, key is test_row_0/A:col10/1731813406776/Put/seqid=0 2024-11-17T03:16:47,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742313_1489 (size=12301) 2024-11-17T03:16:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:47,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. as already flushing 2024-11-17T03:16:48,003 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/197813e56d3f4cc281dc348a89b49de9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/197813e56d3f4cc281dc348a89b49de9 2024-11-17T03:16:48,004 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/c3226fc21f204ecca0823703a59c9065 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c3226fc21f204ecca0823703a59c9065 2024-11-17T03:16:48,007 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into 197813e56d3f4cc281dc348a89b49de9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:48,007 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into c3226fc21f204ecca0823703a59c9065(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:48,007 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=13, startTime=1731813407582; duration=0sec 2024-11-17T03:16:48,007 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=13, startTime=1731813407582; duration=0sec 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:48,007 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:48,007 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:48,008 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6b6e486238fb470d82c8a94b0da74333, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/52c0c487a48b4457b6e5d3905bc9f1e9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/67ebe4d56f684e35a175b2395cda6d3c] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.7 K 2024-11-17T03:16:48,008 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6e486238fb470d82c8a94b0da74333, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731813404103 2024-11-17T03:16:48,008 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52c0c487a48b4457b6e5d3905bc9f1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731813404818 2024-11-17T03:16:48,008 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ebe4d56f684e35a175b2395cda6d3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731813406078 2024-11-17T03:16:48,013 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:48,014 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/5d3a52f1a80f4c7790bf882dfd25427b is 50, key is test_row_0/C:col10/1731813406088/Put/seqid=0 2024-11-17T03:16:48,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742314_1490 (size=13085) 2024-11-17T03:16:48,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813468054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:48,074 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/1e77249deee24dac9995f1cfa81f9ff0 2024-11-17T03:16:48,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b3891d0fbe374255921b8dec9a79f849 is 50, key is test_row_0/B:col10/1731813406776/Put/seqid=0 2024-11-17T03:16:48,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742315_1491 (size=12301) 2024-11-17T03:16:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813468167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:48,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813468377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:48,423 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/5d3a52f1a80f4c7790bf882dfd25427b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5d3a52f1a80f4c7790bf882dfd25427b 2024-11-17T03:16:48,426 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into 5d3a52f1a80f4c7790bf882dfd25427b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:48,426 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:48,426 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=13, startTime=1731813407583; duration=0sec 2024-11-17T03:16:48,426 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:48,426 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:48,484 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b3891d0fbe374255921b8dec9a79f849 2024-11-17T03:16:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/5c63618a64584cda81391918373fd456 is 50, key is test_row_0/C:col10/1731813406776/Put/seqid=0 2024-11-17T03:16:48,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742316_1492 (size=12301) 2024-11-17T03:16:48,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813468697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:48,895 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/5c63618a64584cda81391918373fd456 2024-11-17T03:16:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/1e77249deee24dac9995f1cfa81f9ff0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1e77249deee24dac9995f1cfa81f9ff0 2024-11-17T03:16:48,900 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1e77249deee24dac9995f1cfa81f9ff0, entries=150, sequenceid=347, filesize=12.0 K 2024-11-17T03:16:48,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/b3891d0fbe374255921b8dec9a79f849 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b3891d0fbe374255921b8dec9a79f849 2024-11-17T03:16:48,903 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b3891d0fbe374255921b8dec9a79f849, entries=150, sequenceid=347, filesize=12.0 K 2024-11-17T03:16:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/5c63618a64584cda81391918373fd456 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5c63618a64584cda81391918373fd456 2024-11-17T03:16:48,906 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5c63618a64584cda81391918373fd456, entries=150, sequenceid=347, filesize=12.0 K 2024-11-17T03:16:48,907 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1240ms, sequenceid=347, compaction requested=false 2024-11-17T03:16:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T03:16:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T03:16:48,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T03:16:48,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-11-17T03:16:48,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.3120 sec 2024-11-17T03:16:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:49,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T03:16:49,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:49,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:49,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:49,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:49,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:49,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:49,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/da352f266d06499fa46fbb8e4566e0ed is 50, key is test_row_0/A:col10/1731813408042/Put/seqid=0 2024-11-17T03:16:49,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742317_1493 (size=12301) 2024-11-17T03:16:49,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/da352f266d06499fa46fbb8e4566e0ed 2024-11-17T03:16:49,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7fa7fd07beb349de8f426cefc332fe27 is 50, key is test_row_0/B:col10/1731813408042/Put/seqid=0 2024-11-17T03:16:49,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742318_1494 (size=12301) 2024-11-17T03:16:49,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7fa7fd07beb349de8f426cefc332fe27 2024-11-17T03:16:49,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/7916bb3c5c1b40fcb9b3242f00ced454 is 50, key is test_row_0/C:col10/1731813408042/Put/seqid=0 2024-11-17T03:16:49,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742319_1495 (size=12301) 2024-11-17T03:16:49,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813469280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:49,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:49,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813469397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:49,536 INFO [Thread-1894 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1894. 2024-11-17T03:16:49,536 DEBUG [Thread-1894 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:49,536 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,536 DEBUG [Thread-1894 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:49,542 INFO [Thread-1896 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1896. 2024-11-17T03:16:49,542 DEBUG [Thread-1896 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:49,542 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,542 DEBUG [Thread-1896 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:49,545 INFO [Thread-1902 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1902. 2024-11-17T03:16:49,545 DEBUG [Thread-1902 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:49,545 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,545 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:49,548 INFO [Thread-1900 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1900. 2024-11-17T03:16:49,548 DEBUG [Thread-1900 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:49,548 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,548 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:49,552 INFO [Thread-1898 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1898. 2024-11-17T03:16:49,552 DEBUG [Thread-1898 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:49,552 DEBUG [Thread-1898 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,552 DEBUG [Thread-1898 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44700 deadline: 1731813469617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 , the old value is region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., hostname=695c90f4bb18,38753,1731813266381, seqNum=2 because the exception is null or not the one we care about 2024-11-17T03:16:49,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/7916bb3c5c1b40fcb9b3242f00ced454 2024-11-17T03:16:49,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/da352f266d06499fa46fbb8e4566e0ed as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/da352f266d06499fa46fbb8e4566e0ed 2024-11-17T03:16:49,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/da352f266d06499fa46fbb8e4566e0ed, entries=150, sequenceid=371, filesize=12.0 K 2024-11-17T03:16:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/7fa7fd07beb349de8f426cefc332fe27 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa7fd07beb349de8f426cefc332fe27 2024-11-17T03:16:49,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa7fd07beb349de8f426cefc332fe27, entries=150, sequenceid=371, filesize=12.0 K 2024-11-17T03:16:49,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/7916bb3c5c1b40fcb9b3242f00ced454 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/7916bb3c5c1b40fcb9b3242f00ced454 2024-11-17T03:16:49,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/7916bb3c5c1b40fcb9b3242f00ced454, entries=150, sequenceid=371, filesize=12.0 K 2024-11-17T03:16:49,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ed49cc07eb87ee1fb278a842d5aee3f6 in 454ms, sequenceid=371, compaction requested=true 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:16:49,668 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:49,668 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed49cc07eb87ee1fb278a842d5aee3f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:16:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:49,669 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:49,669 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:49,669 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/A is initiating minor compaction (all files) 2024-11-17T03:16:49,669 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/B is initiating minor compaction (all files) 2024-11-17T03:16:49,669 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/A in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:49,670 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/B in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:49,670 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/197813e56d3f4cc281dc348a89b49de9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1e77249deee24dac9995f1cfa81f9ff0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/da352f266d06499fa46fbb8e4566e0ed] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.8 K 2024-11-17T03:16:49,670 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c3226fc21f204ecca0823703a59c9065, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b3891d0fbe374255921b8dec9a79f849, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa7fd07beb349de8f426cefc332fe27] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.8 K 2024-11-17T03:16:49,670 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 197813e56d3f4cc281dc348a89b49de9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731813406078 2024-11-17T03:16:49,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c3226fc21f204ecca0823703a59c9065, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731813406078 2024-11-17T03:16:49,670 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e77249deee24dac9995f1cfa81f9ff0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731813406766 2024-11-17T03:16:49,670 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b3891d0fbe374255921b8dec9a79f849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731813406766 2024-11-17T03:16:49,671 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting da352f266d06499fa46fbb8e4566e0ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731813408030 2024-11-17T03:16:49,671 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa7fd07beb349de8f426cefc332fe27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731813408030 2024-11-17T03:16:49,678 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:49,679 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/6f3724e516904963a087c6aa0a423ca5 is 50, key is test_row_0/B:col10/1731813408042/Put/seqid=0 2024-11-17T03:16:49,679 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:49,680 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/4800624489e04e5594dfcd8475d18178 is 50, key is test_row_0/A:col10/1731813408042/Put/seqid=0 2024-11-17T03:16:49,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742320_1496 (size=13187) 2024-11-17T03:16:49,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742321_1497 (size=13187) 2024-11-17T03:16:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:49,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:16:49,932 INFO [Thread-1891 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1891. 2024-11-17T03:16:49,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:49,932 DEBUG [Thread-1891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:49,932 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:49,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:49,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:49,932 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:49,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:49,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:49,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:49,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/bcae3ed609e949609846fa86af9e5c4e is 50, key is test_row_0/A:col10/1731813409928/Put/seqid=0 2024-11-17T03:16:49,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742322_1498 (size=12301) 2024-11-17T03:16:50,096 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/6f3724e516904963a087c6aa0a423ca5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/6f3724e516904963a087c6aa0a423ca5 2024-11-17T03:16:50,096 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/4800624489e04e5594dfcd8475d18178 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/4800624489e04e5594dfcd8475d18178 2024-11-17T03:16:50,101 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/B of ed49cc07eb87ee1fb278a842d5aee3f6 into 6f3724e516904963a087c6aa0a423ca5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:50,101 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/A of ed49cc07eb87ee1fb278a842d5aee3f6 into 4800624489e04e5594dfcd8475d18178(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:50,101 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/A, priority=13, startTime=1731813409668; duration=0sec 2024-11-17T03:16:50,101 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/B, priority=13, startTime=1731813409668; duration=0sec 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:B 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:A 2024-11-17T03:16:50,101 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:16:50,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:16:50,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): ed49cc07eb87ee1fb278a842d5aee3f6/C is initiating minor compaction (all files) 2024-11-17T03:16:50,102 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed49cc07eb87ee1fb278a842d5aee3f6/C in TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:50,102 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5d3a52f1a80f4c7790bf882dfd25427b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5c63618a64584cda81391918373fd456, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/7916bb3c5c1b40fcb9b3242f00ced454] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp, totalSize=36.8 K 2024-11-17T03:16:50,102 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3a52f1a80f4c7790bf882dfd25427b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731813406078 2024-11-17T03:16:50,103 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c63618a64584cda81391918373fd456, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731813406766 2024-11-17T03:16:50,103 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7916bb3c5c1b40fcb9b3242f00ced454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731813408030 2024-11-17T03:16:50,111 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed49cc07eb87ee1fb278a842d5aee3f6#C#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:16:50,111 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/64115a8497bd4990b93391343113e2e1 is 50, key is test_row_0/C:col10/1731813408042/Put/seqid=0 2024-11-17T03:16:50,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742323_1499 (size=13187) 2024-11-17T03:16:50,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/bcae3ed609e949609846fa86af9e5c4e 2024-11-17T03:16:50,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f273a976445942009cd0e1f7f82a2dc1 is 50, key is test_row_0/B:col10/1731813409928/Put/seqid=0 2024-11-17T03:16:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742324_1500 (size=12301) 2024-11-17T03:16:50,528 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/64115a8497bd4990b93391343113e2e1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/64115a8497bd4990b93391343113e2e1 2024-11-17T03:16:50,533 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed49cc07eb87ee1fb278a842d5aee3f6/C of ed49cc07eb87ee1fb278a842d5aee3f6 into 64115a8497bd4990b93391343113e2e1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:16:50,533 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:50,533 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., storeName=ed49cc07eb87ee1fb278a842d5aee3f6/C, priority=13, startTime=1731813409668; duration=0sec 2024-11-17T03:16:50,533 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:16:50,534 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed49cc07eb87ee1fb278a842d5aee3f6:C 2024-11-17T03:16:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T03:16:50,758 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:50,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f273a976445942009cd0e1f7f82a2dc1 2024-11-17T03:16:50,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c10a9677d71a4307b2be5fd0560dc37d is 50, key is test_row_0/C:col10/1731813409928/Put/seqid=0 2024-11-17T03:16:50,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742325_1501 (size=12301) 2024-11-17T03:16:50,809 INFO [Thread-1887 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1887. 2024-11-17T03:16:50,809 DEBUG [Thread-1887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:50,809 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:50,810 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:50,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:50,841 INFO [Thread-1889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1889. 2024-11-17T03:16:50,841 DEBUG [Thread-1889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:50,842 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:50,842 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:50,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:51,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c10a9677d71a4307b2be5fd0560dc37d 2024-11-17T03:16:51,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/bcae3ed609e949609846fa86af9e5c4e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/bcae3ed609e949609846fa86af9e5c4e 2024-11-17T03:16:51,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/bcae3ed609e949609846fa86af9e5c4e, entries=150, sequenceid=383, filesize=12.0 K 2024-11-17T03:16:51,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/f273a976445942009cd0e1f7f82a2dc1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f273a976445942009cd0e1f7f82a2dc1 2024-11-17T03:16:51,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f273a976445942009cd0e1f7f82a2dc1, entries=150, sequenceid=383, filesize=12.0 K 2024-11-17T03:16:51,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/c10a9677d71a4307b2be5fd0560dc37d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c10a9677d71a4307b2be5fd0560dc37d 2024-11-17T03:16:51,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c10a9677d71a4307b2be5fd0560dc37d, entries=150, sequenceid=383, filesize=12.0 K 2024-11-17T03:16:51,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1271ms, sequenceid=383, compaction requested=false 2024-11-17T03:16:51,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed49cc07eb87ee1fb278a842d5aee3f6: 2024-11-17T03:16:51,228 INFO [Thread-1883 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1883. 2024-11-17T03:16:51,229 INFO [Thread-1885 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1885. 2024-11-17T03:16:51,229 DEBUG [Thread-1883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:51,229 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:51,229 DEBUG [Thread-1885 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:16:51,229 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:51,229 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:51,229 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:51,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:51,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 167 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1160 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3476 rows 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1153 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3459 rows 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1163 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3488 rows 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1158 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3473 rows 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-17T03:16:51,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3453 rows 2024-11-17T03:16:51,229 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:16:51,229 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:16:51,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:51,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:51,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:16:51,230 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T03:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T03:16:51,233 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813411233"}]},"ts":"1731813411233"} 2024-11-17T03:16:51,234 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T03:16:51,234 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T03:16:51,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:16:51,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, UNASSIGN}] 2024-11-17T03:16:51,236 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, UNASSIGN 2024-11-17T03:16:51,236 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ed49cc07eb87ee1fb278a842d5aee3f6, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:51,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, UNASSIGN because future has completed 2024-11-17T03:16:51,237 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:16:51,238 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T03:16:51,390 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:51,390 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:16:51,390 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing ed49cc07eb87ee1fb278a842d5aee3f6, disabling compactions & flushes 2024-11-17T03:16:51,390 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:51,390 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:51,391 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. after waiting 0 ms 2024-11-17T03:16:51,391 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:51,391 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing ed49cc07eb87ee1fb278a842d5aee3f6 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T03:16:51,391 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=A 2024-11-17T03:16:51,391 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:51,392 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=B 2024-11-17T03:16:51,392 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:51,392 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ed49cc07eb87ee1fb278a842d5aee3f6, store=C 2024-11-17T03:16:51,392 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:51,398 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/76da1313c9f444f88278f495bf9e6394 is 50, key is test_row_1/A:col10/1731813410838/Put/seqid=0 2024-11-17T03:16:51,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742326_1502 (size=9857) 2024-11-17T03:16:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T03:16:51,805 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/76da1313c9f444f88278f495bf9e6394 2024-11-17T03:16:51,816 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/be4f27510629479283c83c003978bae8 is 50, key is test_row_1/B:col10/1731813410838/Put/seqid=0 2024-11-17T03:16:51,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742327_1503 (size=9857) 2024-11-17T03:16:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T03:16:52,220 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/be4f27510629479283c83c003978bae8 2024-11-17T03:16:52,233 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/71002604f76945d09cd3c647a7ee1dd2 is 50, key is test_row_1/C:col10/1731813410838/Put/seqid=0 2024-11-17T03:16:52,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742328_1504 (size=9857) 2024-11-17T03:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T03:16:52,640 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/71002604f76945d09cd3c647a7ee1dd2 2024-11-17T03:16:52,649 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/A/76da1313c9f444f88278f495bf9e6394 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/76da1313c9f444f88278f495bf9e6394 2024-11-17T03:16:52,656 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/76da1313c9f444f88278f495bf9e6394, entries=100, sequenceid=393, filesize=9.6 K 2024-11-17T03:16:52,657 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/B/be4f27510629479283c83c003978bae8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/be4f27510629479283c83c003978bae8 2024-11-17T03:16:52,660 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/be4f27510629479283c83c003978bae8, entries=100, sequenceid=393, filesize=9.6 K 2024-11-17T03:16:52,661 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/.tmp/C/71002604f76945d09cd3c647a7ee1dd2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/71002604f76945d09cd3c647a7ee1dd2 2024-11-17T03:16:52,664 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/71002604f76945d09cd3c647a7ee1dd2, entries=100, sequenceid=393, filesize=9.6 K 2024-11-17T03:16:52,665 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1274ms, sequenceid=393, compaction requested=true 2024-11-17T03:16:52,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/d20c425c12f64463b09ce38c1da8ade0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/2c372f58b8fe466cac8b14c65dc90936, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c01db0aaf176411c8289cd1bc7c451fc, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/6cbcf3b7d84148c6bc687bd4ae8a7214, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/647830beabad486fa8662770ec812076, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fb9ea70b1e074c68915e5fe2390ad500, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fac953eec50b4c93b3bd43d4ea559843, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/57285d957a5143ab9b04217d5f2aedf9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/41d4b7e948164301a1b721b3fdbde887, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/27eaa45eb514471b9e8e2dc1fe317531, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c6a0d47cec654c82b4b5e3f0c6b5a371, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/be46587446464806b9f46a292dcc4230, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/14d228e31fa343e7ad741ae455c2c419, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/e3fa182ff2e04cc8b703ed51dd3812fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cc258e8524cf4efab0620a21f13a7402, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/3d11991565d443aaac0acfd3eabcfe3b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1c8f1400c5e64bb78f9d1675273a5b92, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/ca81fab0a12244d2acb63e66dfc12e7a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/652a1c00c0de436f8fe87d4193031582, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/150672502d614589a038b601f8c2de16, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/f3bb649f51e04578a0be682c72f624d9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/02919486bcc740a9b7cab48dc7a8f78e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cd58efec8dfd4854a19bf5a61d6fc6c1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/197813e56d3f4cc281dc348a89b49de9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1e77249deee24dac9995f1cfa81f9ff0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/da352f266d06499fa46fbb8e4566e0ed] to archive 2024-11-17T03:16:52,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:16:52,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/d20c425c12f64463b09ce38c1da8ade0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/d20c425c12f64463b09ce38c1da8ade0 2024-11-17T03:16:52,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/2c372f58b8fe466cac8b14c65dc90936 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/2c372f58b8fe466cac8b14c65dc90936 2024-11-17T03:16:52,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c01db0aaf176411c8289cd1bc7c451fc to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c01db0aaf176411c8289cd1bc7c451fc 2024-11-17T03:16:52,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/6cbcf3b7d84148c6bc687bd4ae8a7214 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/6cbcf3b7d84148c6bc687bd4ae8a7214 2024-11-17T03:16:52,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/647830beabad486fa8662770ec812076 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/647830beabad486fa8662770ec812076 2024-11-17T03:16:52,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fb9ea70b1e074c68915e5fe2390ad500 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fb9ea70b1e074c68915e5fe2390ad500 2024-11-17T03:16:52,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fac953eec50b4c93b3bd43d4ea559843 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/fac953eec50b4c93b3bd43d4ea559843 2024-11-17T03:16:52,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/57285d957a5143ab9b04217d5f2aedf9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/57285d957a5143ab9b04217d5f2aedf9 2024-11-17T03:16:52,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/41d4b7e948164301a1b721b3fdbde887 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/41d4b7e948164301a1b721b3fdbde887 2024-11-17T03:16:52,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/27eaa45eb514471b9e8e2dc1fe317531 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/27eaa45eb514471b9e8e2dc1fe317531 2024-11-17T03:16:52,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c6a0d47cec654c82b4b5e3f0c6b5a371 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/c6a0d47cec654c82b4b5e3f0c6b5a371 2024-11-17T03:16:52,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/be46587446464806b9f46a292dcc4230 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/be46587446464806b9f46a292dcc4230 2024-11-17T03:16:52,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/14d228e31fa343e7ad741ae455c2c419 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/14d228e31fa343e7ad741ae455c2c419 2024-11-17T03:16:52,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/e3fa182ff2e04cc8b703ed51dd3812fd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/e3fa182ff2e04cc8b703ed51dd3812fd 2024-11-17T03:16:52,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cc258e8524cf4efab0620a21f13a7402 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cc258e8524cf4efab0620a21f13a7402 2024-11-17T03:16:52,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/3d11991565d443aaac0acfd3eabcfe3b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/3d11991565d443aaac0acfd3eabcfe3b 2024-11-17T03:16:52,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1c8f1400c5e64bb78f9d1675273a5b92 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1c8f1400c5e64bb78f9d1675273a5b92 2024-11-17T03:16:52,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/ca81fab0a12244d2acb63e66dfc12e7a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/ca81fab0a12244d2acb63e66dfc12e7a 2024-11-17T03:16:52,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/652a1c00c0de436f8fe87d4193031582 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/652a1c00c0de436f8fe87d4193031582 2024-11-17T03:16:52,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/150672502d614589a038b601f8c2de16 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/150672502d614589a038b601f8c2de16 2024-11-17T03:16:52,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/f3bb649f51e04578a0be682c72f624d9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/f3bb649f51e04578a0be682c72f624d9 2024-11-17T03:16:52,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/02919486bcc740a9b7cab48dc7a8f78e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/02919486bcc740a9b7cab48dc7a8f78e 2024-11-17T03:16:52,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cd58efec8dfd4854a19bf5a61d6fc6c1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/cd58efec8dfd4854a19bf5a61d6fc6c1 2024-11-17T03:16:52,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/197813e56d3f4cc281dc348a89b49de9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/197813e56d3f4cc281dc348a89b49de9 2024-11-17T03:16:52,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1e77249deee24dac9995f1cfa81f9ff0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/1e77249deee24dac9995f1cfa81f9ff0 2024-11-17T03:16:52,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/da352f266d06499fa46fbb8e4566e0ed to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/da352f266d06499fa46fbb8e4566e0ed 2024-11-17T03:16:52,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f1fd8a210a004a008252a0cf570e6b9f, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c369b8f2294d455d8318d3f9fd2b0d00, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/cca1be76e8c74665bba89bbb0f2babfa, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b386a41d89b04428af6fb5545ebd363e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/0e579893441f4f4aabf999531b5ee2cf, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b37769c5c4c44cda971440fa722dbeeb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7b3ba0395e5648e1b5637abb6837ba57, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2908202520e743f28564ac5a34363b43, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f054dda7ddc849098ba835b60874814b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2c872d6728eb435e9d66667b3f230bbd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c804e92becee4331950c8f96ad3cd652, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa3b5384b0d4378b67df15b7c68c7fe, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/858a6a4e4cf245759a846f5ab271065e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a24bbf63d4644b5a824681cdebc564e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/02db5be115084678b0ae0aee9ac96627, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/58108cb3412f43879fed2a7e8b246a7c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/eb258ff544c44caeaa21519d89725ff5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f8914cfd236c448c9e1d6561145b59e2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/d744affaeee0472b951304dac966ad3d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7ac3aa48767b44449d65d55ee4eb5c11, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/391f0245643042ceb55f92d090c14bf5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/e1eca82ff1874c0e93cc5933d7372702, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c3226fc21f204ecca0823703a59c9065, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a0e1e135aef4459b4e9d35b592de6fd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b3891d0fbe374255921b8dec9a79f849, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa7fd07beb349de8f426cefc332fe27] to archive 2024-11-17T03:16:52,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:16:52,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f1fd8a210a004a008252a0cf570e6b9f to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f1fd8a210a004a008252a0cf570e6b9f 2024-11-17T03:16:52,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c369b8f2294d455d8318d3f9fd2b0d00 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c369b8f2294d455d8318d3f9fd2b0d00 2024-11-17T03:16:52,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/cca1be76e8c74665bba89bbb0f2babfa to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/cca1be76e8c74665bba89bbb0f2babfa 2024-11-17T03:16:52,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b386a41d89b04428af6fb5545ebd363e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b386a41d89b04428af6fb5545ebd363e 2024-11-17T03:16:52,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/0e579893441f4f4aabf999531b5ee2cf to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/0e579893441f4f4aabf999531b5ee2cf 2024-11-17T03:16:52,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b37769c5c4c44cda971440fa722dbeeb to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b37769c5c4c44cda971440fa722dbeeb 2024-11-17T03:16:52,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7b3ba0395e5648e1b5637abb6837ba57 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7b3ba0395e5648e1b5637abb6837ba57 2024-11-17T03:16:52,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2908202520e743f28564ac5a34363b43 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2908202520e743f28564ac5a34363b43 2024-11-17T03:16:52,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f054dda7ddc849098ba835b60874814b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f054dda7ddc849098ba835b60874814b 2024-11-17T03:16:52,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2c872d6728eb435e9d66667b3f230bbd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/2c872d6728eb435e9d66667b3f230bbd 2024-11-17T03:16:52,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c804e92becee4331950c8f96ad3cd652 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c804e92becee4331950c8f96ad3cd652 2024-11-17T03:16:52,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa3b5384b0d4378b67df15b7c68c7fe to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa3b5384b0d4378b67df15b7c68c7fe 2024-11-17T03:16:52,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/858a6a4e4cf245759a846f5ab271065e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/858a6a4e4cf245759a846f5ab271065e 2024-11-17T03:16:52,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a24bbf63d4644b5a824681cdebc564e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a24bbf63d4644b5a824681cdebc564e 2024-11-17T03:16:52,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/02db5be115084678b0ae0aee9ac96627 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/02db5be115084678b0ae0aee9ac96627 2024-11-17T03:16:52,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/58108cb3412f43879fed2a7e8b246a7c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/58108cb3412f43879fed2a7e8b246a7c 2024-11-17T03:16:52,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/eb258ff544c44caeaa21519d89725ff5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/eb258ff544c44caeaa21519d89725ff5 2024-11-17T03:16:52,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f8914cfd236c448c9e1d6561145b59e2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f8914cfd236c448c9e1d6561145b59e2 2024-11-17T03:16:52,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/d744affaeee0472b951304dac966ad3d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/d744affaeee0472b951304dac966ad3d 2024-11-17T03:16:52,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7ac3aa48767b44449d65d55ee4eb5c11 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7ac3aa48767b44449d65d55ee4eb5c11 2024-11-17T03:16:52,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/391f0245643042ceb55f92d090c14bf5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/391f0245643042ceb55f92d090c14bf5 2024-11-17T03:16:52,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/e1eca82ff1874c0e93cc5933d7372702 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/e1eca82ff1874c0e93cc5933d7372702 2024-11-17T03:16:52,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c3226fc21f204ecca0823703a59c9065 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/c3226fc21f204ecca0823703a59c9065 2024-11-17T03:16:52,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a0e1e135aef4459b4e9d35b592de6fd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/8a0e1e135aef4459b4e9d35b592de6fd 2024-11-17T03:16:52,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b3891d0fbe374255921b8dec9a79f849 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/b3891d0fbe374255921b8dec9a79f849 2024-11-17T03:16:52,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa7fd07beb349de8f426cefc332fe27 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/7fa7fd07beb349de8f426cefc332fe27 2024-11-17T03:16:52,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ef75f04e57934e058c31fc04d29ac9a6, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6625eb373a364b529b431aaf740983a0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/3cfe777c7fbe48778649ec5eed7935ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/be577d434914424c811d9cbd7dc55233, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/69d9b03d02a047b5a66e9ffa506633f3, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/4462cad6956d4dcf87f902c279e9bb80, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c5c37f1778de4ed7b4d4cb056a29bf0d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/f7256558db9e4520a847cda214e8dddd, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6dad1b2432f74687b1ace79476abffc8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/a2a9859dddac442192e064d6311e8c44, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ce7b62bc6b54485785fa8497b8ce2002, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/b516f2cf29834bb5bd822134c9af25a1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81b5649d209442249b3f7efb5bdf26c7, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81ec0d79091b47b7887e35feb8b10754, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/bc404b765af040f78145419e43efb2a2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/9146c5d6d3bb446fa282dccdeb7983ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e292a951eba449d7a06077c2ebad0a6c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c47c3eac5ed443d6ad717511dfa20c65, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/353d55a162104d49b5844af3857ba384, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6b6e486238fb470d82c8a94b0da74333, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e15c14fc29354634b562176b8e352be5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/52c0c487a48b4457b6e5d3905bc9f1e9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5d3a52f1a80f4c7790bf882dfd25427b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/67ebe4d56f684e35a175b2395cda6d3c, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5c63618a64584cda81391918373fd456, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/7916bb3c5c1b40fcb9b3242f00ced454] to archive 2024-11-17T03:16:52,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:16:52,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ef75f04e57934e058c31fc04d29ac9a6 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ef75f04e57934e058c31fc04d29ac9a6 2024-11-17T03:16:52,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6625eb373a364b529b431aaf740983a0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6625eb373a364b529b431aaf740983a0 2024-11-17T03:16:52,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/3cfe777c7fbe48778649ec5eed7935ff to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/3cfe777c7fbe48778649ec5eed7935ff 2024-11-17T03:16:52,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/be577d434914424c811d9cbd7dc55233 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/be577d434914424c811d9cbd7dc55233 2024-11-17T03:16:52,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/69d9b03d02a047b5a66e9ffa506633f3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/69d9b03d02a047b5a66e9ffa506633f3 2024-11-17T03:16:52,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/4462cad6956d4dcf87f902c279e9bb80 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/4462cad6956d4dcf87f902c279e9bb80 2024-11-17T03:16:52,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c5c37f1778de4ed7b4d4cb056a29bf0d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c5c37f1778de4ed7b4d4cb056a29bf0d 2024-11-17T03:16:52,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/f7256558db9e4520a847cda214e8dddd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/f7256558db9e4520a847cda214e8dddd 2024-11-17T03:16:52,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6dad1b2432f74687b1ace79476abffc8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6dad1b2432f74687b1ace79476abffc8 2024-11-17T03:16:52,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/a2a9859dddac442192e064d6311e8c44 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/a2a9859dddac442192e064d6311e8c44 2024-11-17T03:16:52,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ce7b62bc6b54485785fa8497b8ce2002 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/ce7b62bc6b54485785fa8497b8ce2002 2024-11-17T03:16:52,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/b516f2cf29834bb5bd822134c9af25a1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/b516f2cf29834bb5bd822134c9af25a1 2024-11-17T03:16:52,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81b5649d209442249b3f7efb5bdf26c7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81b5649d209442249b3f7efb5bdf26c7 2024-11-17T03:16:52,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81ec0d79091b47b7887e35feb8b10754 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/81ec0d79091b47b7887e35feb8b10754 2024-11-17T03:16:52,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/bc404b765af040f78145419e43efb2a2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/bc404b765af040f78145419e43efb2a2 2024-11-17T03:16:52,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/9146c5d6d3bb446fa282dccdeb7983ff to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/9146c5d6d3bb446fa282dccdeb7983ff 2024-11-17T03:16:52,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e292a951eba449d7a06077c2ebad0a6c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e292a951eba449d7a06077c2ebad0a6c 2024-11-17T03:16:52,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c47c3eac5ed443d6ad717511dfa20c65 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c47c3eac5ed443d6ad717511dfa20c65 2024-11-17T03:16:52,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/353d55a162104d49b5844af3857ba384 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/353d55a162104d49b5844af3857ba384 2024-11-17T03:16:52,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6b6e486238fb470d82c8a94b0da74333 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/6b6e486238fb470d82c8a94b0da74333 2024-11-17T03:16:52,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e15c14fc29354634b562176b8e352be5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/e15c14fc29354634b562176b8e352be5 2024-11-17T03:16:52,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/52c0c487a48b4457b6e5d3905bc9f1e9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/52c0c487a48b4457b6e5d3905bc9f1e9 2024-11-17T03:16:52,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5d3a52f1a80f4c7790bf882dfd25427b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5d3a52f1a80f4c7790bf882dfd25427b 2024-11-17T03:16:52,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/67ebe4d56f684e35a175b2395cda6d3c to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/67ebe4d56f684e35a175b2395cda6d3c 2024-11-17T03:16:52,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5c63618a64584cda81391918373fd456 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/5c63618a64584cda81391918373fd456 2024-11-17T03:16:52,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/7916bb3c5c1b40fcb9b3242f00ced454 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/7916bb3c5c1b40fcb9b3242f00ced454 2024-11-17T03:16:52,755 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=1 2024-11-17T03:16:52,756 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. 2024-11-17T03:16:52,756 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for ed49cc07eb87ee1fb278a842d5aee3f6: Waiting for close lock at 1731813411390Running coprocessor pre-close hooks at 1731813411390Disabling compacts and flushes for region at 1731813411390Disabling writes for close at 1731813411391 (+1 ms)Obtaining lock to block concurrent updates at 1731813411391Preparing flush snapshotting stores in ed49cc07eb87ee1fb278a842d5aee3f6 at 1731813411391Finished memstore snapshotting TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731813411392 (+1 ms)Flushing stores of TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6. at 1731813411393 (+1 ms)Flushing ed49cc07eb87ee1fb278a842d5aee3f6/A: creating writer at 1731813411393Flushing ed49cc07eb87ee1fb278a842d5aee3f6/A: appending metadata at 1731813411397 (+4 ms)Flushing ed49cc07eb87ee1fb278a842d5aee3f6/A: closing flushed file at 1731813411397Flushing ed49cc07eb87ee1fb278a842d5aee3f6/B: creating writer at 1731813411813 (+416 ms)Flushing ed49cc07eb87ee1fb278a842d5aee3f6/B: appending metadata at 1731813411816 (+3 ms)Flushing ed49cc07eb87ee1fb278a842d5aee3f6/B: closing flushed file at 1731813411816Flushing ed49cc07eb87ee1fb278a842d5aee3f6/C: creating writer at 1731813412228 (+412 ms)Flushing ed49cc07eb87ee1fb278a842d5aee3f6/C: appending metadata at 1731813412232 (+4 ms)Flushing ed49cc07eb87ee1fb278a842d5aee3f6/C: closing flushed file at 1731813412232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@576fcf8d: reopening flushed file at 1731813412648 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53fc373a: reopening flushed file at 1731813412656 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f878682: reopening flushed file at 1731813412660 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ed49cc07eb87ee1fb278a842d5aee3f6 in 1274ms, sequenceid=393, compaction requested=true at 1731813412665 (+5 ms)Writing region close event to WAL at 1731813412751 (+86 ms)Running coprocessor post-close hooks at 1731813412756 (+5 ms)Closed at 1731813412756 2024-11-17T03:16:52,759 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:52,759 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ed49cc07eb87ee1fb278a842d5aee3f6, regionState=CLOSED 2024-11-17T03:16:52,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:52,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T03:16:52,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure ed49cc07eb87ee1fb278a842d5aee3f6, server=695c90f4bb18,38753,1731813266381 in 1.5240 sec 2024-11-17T03:16:52,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-17T03:16:52,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed49cc07eb87ee1fb278a842d5aee3f6, UNASSIGN in 1.5280 sec 2024-11-17T03:16:52,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T03:16:52,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5310 sec 2024-11-17T03:16:52,767 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813412767"}]},"ts":"1731813412767"} 2024-11-17T03:16:52,768 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T03:16:52,768 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T03:16:52,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5390 sec 2024-11-17T03:16:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T03:16:53,378 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:53,380 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T03:16:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,384 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T03:16:53,385 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,388 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:53,391 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/recovered.edits] 2024-11-17T03:16:53,395 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/4800624489e04e5594dfcd8475d18178 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/4800624489e04e5594dfcd8475d18178 2024-11-17T03:16:53,396 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/76da1313c9f444f88278f495bf9e6394 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/76da1313c9f444f88278f495bf9e6394 2024-11-17T03:16:53,398 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/bcae3ed609e949609846fa86af9e5c4e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/A/bcae3ed609e949609846fa86af9e5c4e 2024-11-17T03:16:53,401 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/6f3724e516904963a087c6aa0a423ca5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/6f3724e516904963a087c6aa0a423ca5 2024-11-17T03:16:53,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/be4f27510629479283c83c003978bae8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/be4f27510629479283c83c003978bae8 2024-11-17T03:16:53,404 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f273a976445942009cd0e1f7f82a2dc1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/B/f273a976445942009cd0e1f7f82a2dc1 2024-11-17T03:16:53,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/64115a8497bd4990b93391343113e2e1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/64115a8497bd4990b93391343113e2e1 2024-11-17T03:16:53,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/71002604f76945d09cd3c647a7ee1dd2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/71002604f76945d09cd3c647a7ee1dd2 2024-11-17T03:16:53,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c10a9677d71a4307b2be5fd0560dc37d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/C/c10a9677d71a4307b2be5fd0560dc37d 2024-11-17T03:16:53,410 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/recovered.edits/396.seqid to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6/recovered.edits/396.seqid 2024-11-17T03:16:53,411 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/ed49cc07eb87ee1fb278a842d5aee3f6 2024-11-17T03:16:53,411 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T03:16:53,413 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,415 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T03:16:53,417 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T03:16:53,418 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,418 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T03:16:53,418 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731813413418"}]},"ts":"9223372036854775807"} 2024-11-17T03:16:53,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T03:16:53,420 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ed49cc07eb87ee1fb278a842d5aee3f6, NAME => 'TestAcidGuarantees,,1731813387335.ed49cc07eb87ee1fb278a842d5aee3f6.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T03:16:53,420 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T03:16:53,420 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731813413420"}]},"ts":"9223372036854775807"} 2024-11-17T03:16:53,422 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T03:16:53,423 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-17T03:16:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T03:16:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T03:16:53,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:53,516 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=462 (was 450) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1067 (was 1078) 2024-11-17T03:16:53,528 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=462, ProcessCount=11, AvailableMemoryMB=1066 2024-11-17T03:16:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:16:53,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:16:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:53,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T03:16:53,531 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:53,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-17T03:16:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T03:16:53,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T03:16:53,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742329_1505 (size=1061) 2024-11-17T03:16:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T03:16:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T03:16:53,940 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64 2024-11-17T03:16:53,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742330_1506 (size=53) 2024-11-17T03:16:54,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T03:16:54,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:16:54,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 97cf29522eb018ba919d51f502de21d3, disabling compactions & flushes 2024-11-17T03:16:54,347 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. after waiting 0 ms 2024-11-17T03:16:54,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,348 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 97cf29522eb018ba919d51f502de21d3: Waiting for close lock at 1731813414347Disabling compacts and flushes for region at 1731813414347Disabling writes for close at 1731813414347Writing region close event to WAL at 1731813414347Closed at 1731813414348 (+1 ms) 2024-11-17T03:16:54,350 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T03:16:54,351 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731813414350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731813414350"}]},"ts":"1731813414350"} 2024-11-17T03:16:54,353 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T03:16:54,355 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T03:16:54,357 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T03:16:54,357 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813414357"}]},"ts":"1731813414357"} 2024-11-17T03:16:54,360 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T03:16:54,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, ASSIGN}] 2024-11-17T03:16:54,361 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, ASSIGN 2024-11-17T03:16:54,362 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, ASSIGN; state=OFFLINE, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=false 2024-11-17T03:16:54,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:54,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, ASSIGN because future has completed 2024-11-17T03:16:54,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T03:16:54,675 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,676 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:16:54,676 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,676 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:16:54,677 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,677 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,679 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,682 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:54,683 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97cf29522eb018ba919d51f502de21d3 columnFamilyName A 2024-11-17T03:16:54,683 DEBUG [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:54,684 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(327): Store=97cf29522eb018ba919d51f502de21d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:54,684 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,686 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:54,686 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97cf29522eb018ba919d51f502de21d3 columnFamilyName B 2024-11-17T03:16:54,686 DEBUG [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:54,687 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(327): Store=97cf29522eb018ba919d51f502de21d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:54,687 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,688 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:54,688 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97cf29522eb018ba919d51f502de21d3 columnFamilyName C 2024-11-17T03:16:54,688 DEBUG [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:54,689 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(327): Store=97cf29522eb018ba919d51f502de21d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:54,689 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,689 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,690 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,690 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,691 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,691 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,692 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:16:54,693 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,695 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T03:16:54,695 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 97cf29522eb018ba919d51f502de21d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69734893, jitterRate=0.03913088142871857}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:16:54,695 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:54,696 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 97cf29522eb018ba919d51f502de21d3: Running coprocessor pre-open hook at 1731813414677Writing region info on filesystem at 1731813414677Initializing all the Stores at 1731813414678 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813414679 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813414679Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813414679Cleaning up temporary data from old regions at 1731813414691 (+12 ms)Running coprocessor post-open hooks at 1731813414695 (+4 ms)Region opened successfully at 1731813414696 (+1 ms) 2024-11-17T03:16:54,697 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., pid=142, masterSystemTime=1731813414671 2024-11-17T03:16:54,699 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,699 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:54,699 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=OPEN, openSeqNum=2, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:54,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:54,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-17T03:16:54,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 in 185 msec 2024-11-17T03:16:54,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-17T03:16:54,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, ASSIGN in 344 msec 2024-11-17T03:16:54,707 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T03:16:54,708 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813414707"}]},"ts":"1731813414707"} 2024-11-17T03:16:54,709 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T03:16:54,711 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T03:16:54,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-17T03:16:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T03:16:55,678 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:55,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eedb90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:55,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:55,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:55,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1759344e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:55,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ed33f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:55,686 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:55,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:55,690 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T03:16:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 695c90f4bb18,42091,1731813265296 2024-11-17T03:16:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f83574d 2024-11-17T03:16:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T03:16:55,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T03:16:55,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T03:16:55,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T03:16:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T03:16:55,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742331_1507 (size=1097) 2024-11-17T03:16:56,113 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T03:16:56,113 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T03:16:56,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:16:56,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, REOPEN/MOVE}] 2024-11-17T03:16:56,122 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, REOPEN/MOVE 2024-11-17T03:16:56,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, REOPEN/MOVE because future has completed 2024-11-17T03:16:56,126 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:16:56,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:56,280 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,280 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:16:56,280 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 97cf29522eb018ba919d51f502de21d3, disabling compactions & flushes 2024-11-17T03:16:56,280 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,281 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,281 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. after waiting 0 ms 2024-11-17T03:16:56,281 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,286 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T03:16:56,287 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,287 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 97cf29522eb018ba919d51f502de21d3: Waiting for close lock at 1731813416280Running coprocessor pre-close hooks at 1731813416280Disabling compacts and flushes for region at 1731813416280Disabling writes for close at 1731813416281 (+1 ms)Writing region close event to WAL at 1731813416282 (+1 ms)Running coprocessor post-close hooks at 1731813416287 (+5 ms)Closed at 1731813416287 2024-11-17T03:16:56,287 WARN [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 97cf29522eb018ba919d51f502de21d3 to self. 2024-11-17T03:16:56,289 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,289 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=CLOSED 2024-11-17T03:16:56,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:56,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T03:16:56,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 in 165 msec 2024-11-17T03:16:56,293 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, REOPEN/MOVE; state=CLOSED, location=695c90f4bb18,38753,1731813266381; forceNewPlan=false, retain=true 2024-11-17T03:16:56,444 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=OPENING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, REOPEN/MOVE because future has completed 2024-11-17T03:16:56,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:16:56,660 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,660 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} 2024-11-17T03:16:56,661 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,661 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T03:16:56,661 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,661 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,664 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,665 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:56,665 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97cf29522eb018ba919d51f502de21d3 columnFamilyName A 2024-11-17T03:16:56,666 DEBUG [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:56,666 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(327): Store=97cf29522eb018ba919d51f502de21d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:56,667 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,667 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:56,667 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97cf29522eb018ba919d51f502de21d3 columnFamilyName B 2024-11-17T03:16:56,667 DEBUG [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:56,668 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(327): Store=97cf29522eb018ba919d51f502de21d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:56,668 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,668 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T03:16:56,669 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97cf29522eb018ba919d51f502de21d3 columnFamilyName C 2024-11-17T03:16:56,669 DEBUG [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:56,669 INFO [StoreOpener-97cf29522eb018ba919d51f502de21d3-1 {}] regionserver.HStore(327): Store=97cf29522eb018ba919d51f502de21d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T03:16:56,669 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,669 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,670 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,671 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,672 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,672 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,672 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T03:16:56,674 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,675 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 97cf29522eb018ba919d51f502de21d3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61039777, jitterRate=-0.09043644368648529}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T03:16:56,675 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,675 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 97cf29522eb018ba919d51f502de21d3: Running coprocessor pre-open hook at 1731813416661Writing region info on filesystem at 1731813416661Initializing all the Stores at 1731813416663 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813416663Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813416664 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731813416664Cleaning up temporary data from old regions at 1731813416672 (+8 ms)Running coprocessor post-open hooks at 1731813416675 (+3 ms)Region opened successfully at 1731813416675 2024-11-17T03:16:56,676 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., pid=147, masterSystemTime=1731813416654 2024-11-17T03:16:56,678 DEBUG [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,678 INFO [RS_OPEN_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,678 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=OPEN, openSeqNum=5, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:16:56,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-17T03:16:56,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 in 232 msec 2024-11-17T03:16:56,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-17T03:16:56,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, REOPEN/MOVE in 561 msec 2024-11-17T03:16:56,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T03:16:56,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 566 msec 2024-11-17T03:16:56,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 992 msec 2024-11-17T03:16:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T03:16:56,689 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:16:56,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bc30a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,691 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4029972d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12015da8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227ae0e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,695 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad98dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,697 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387eb9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401e798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c50d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a0f042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2ee11c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f054324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,705 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202ee0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cbdd75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,708 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d218e26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465ad5ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1001aa8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,712 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e11f3d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57184373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7bff21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5822c562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5891c374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2daf9160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,719 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb896d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afce071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6719bbfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e4dcf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119944f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611845f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 695c90f4bb18,42091,-1 for getting cluster id 2024-11-17T03:16:56,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T03:16:56,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d5edb17-01db-469a-ad88-13203f32e6ee' 2024-11-17T03:16:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T03:16:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d5edb17-01db-469a-ad88-13203f32e6ee" 2024-11-17T03:16:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ec9427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [695c90f4bb18,42091,-1] 2024-11-17T03:16:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T03:16:56,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:16:56,728 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T03:16:56,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b48af73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T03:16:56,729 DEBUG [Thread-2277 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,729 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,732 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,733 DEBUG [Thread-2286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,733 DEBUG [Thread-2275 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:56,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T03:16:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:16:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:16:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:16:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:56,744 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,745 DEBUG [Thread-2290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,745 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,745 DEBUG [Thread-2279 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,747 DEBUG [Thread-2288 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T03:16:56,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:16:56,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,748 DEBUG [Thread-2279 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=695c90f4bb18,38753,1731813266381, seqNum=-1] 2024-11-17T03:16:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,749 DEBUG [Thread-2279 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T03:16:56,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-17T03:16:56,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T03:16:56,750 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:16:56,751 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:16:56,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:16:56,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:16:56,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5] 2024-11-17T03:16:56,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e7f71be590cb41c282b6fbe59a60d37b_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813416740/Put/seqid=0 2024-11-17T03:16:56,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813416754, max timestamp=1731813416764Num uniques 859; 2024-11-17T03:16:56,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813416754, max timestamp=1731813416764Num uniques 859; 2024-11-17T03:16:56,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:16:56,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:16:56,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731813416754, max timestamp=1731813416764Num uniques 859; 2024-11-17T03:16:56,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/695c90f4bb18:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T03:16:56,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742332_1508 (size=12154) 2024-11-17T03:16:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813476777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813476778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813476778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813476779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813476782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:16:56,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813476887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813476887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813476887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813476887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813476897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:56,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:56,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:56,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:16:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813477097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813477097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813477097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813477097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813477106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:57,177 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e7f71be590cb41c282b6fbe59a60d37b_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e7f71be590cb41c282b6fbe59a60d37b_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:57,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/5b00e60e291a44758ab0bd14d69ce843, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:16:57,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/5b00e60e291a44758ab0bd14d69ce843 is 175, key is test_row_0/A:col10/1731813416740/Put/seqid=0 2024-11-17T03:16:57,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742333_1509 (size=30955) 2024-11-17T03:16:57,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:16:57,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813477407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813477407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813477407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813477407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813477417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/5b00e60e291a44758ab0bd14d69ce843 2024-11-17T03:16:57,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/59fdc51486a34f38bb2618fbcd81b4f2 is 50, key is test_row_0/B:col10/1731813416740/Put/seqid=0 2024-11-17T03:16:57,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742334_1510 (size=12001) 2024-11-17T03:16:57,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:16:57,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813477916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813477917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813477917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813477917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813477927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:57,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:57,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:57,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:57,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:16:58,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/59fdc51486a34f38bb2618fbcd81b4f2 2024-11-17T03:16:58,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/780d37160043418e90ed5c257ce172c0 is 50, key is test_row_0/C:col10/1731813416740/Put/seqid=0 2024-11-17T03:16:58,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742335_1511 (size=12001) 2024-11-17T03:16:58,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/780d37160043418e90ed5c257ce172c0 2024-11-17T03:16:58,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/5b00e60e291a44758ab0bd14d69ce843 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843 2024-11-17T03:16:58,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843, entries=150, sequenceid=15, filesize=30.2 K 2024-11-17T03:16:58,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/59fdc51486a34f38bb2618fbcd81b4f2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59fdc51486a34f38bb2618fbcd81b4f2 2024-11-17T03:16:58,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59fdc51486a34f38bb2618fbcd81b4f2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:16:58,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/780d37160043418e90ed5c257ce172c0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/780d37160043418e90ed5c257ce172c0 2024-11-17T03:16:58,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/780d37160043418e90ed5c257ce172c0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T03:16:58,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 97cf29522eb018ba919d51f502de21d3 in 1310ms, sequenceid=15, compaction requested=false 2024-11-17T03:16:58,052 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T03:16:58,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:16:58,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T03:16:58,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:58,124 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-17T03:16:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:16:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:16:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:16:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:16:58,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b586fb2e553842388c356397b6dc33e4_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813416777/Put/seqid=0 2024-11-17T03:16:58,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742336_1512 (size=12154) 2024-11-17T03:16:58,271 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T03:16:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:58,548 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b586fb2e553842388c356397b6dc33e4_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b586fb2e553842388c356397b6dc33e4_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9b0f0b4f36741c0bf093dd2acbf5355, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:16:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9b0f0b4f36741c0bf093dd2acbf5355 is 175, key is test_row_0/A:col10/1731813416777/Put/seqid=0 2024-11-17T03:16:58,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742337_1513 (size=30955) 2024-11-17T03:16:58,560 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9b0f0b4f36741c0bf093dd2acbf5355 2024-11-17T03:16:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/26d02481005840eb982945f6fd3bd6d0 is 50, key is test_row_0/B:col10/1731813416777/Put/seqid=0 2024-11-17T03:16:58,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742338_1514 (size=12001) 2024-11-17T03:16:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:16:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:58,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:16:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813478938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:58,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813478939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:58,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813478941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813478943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813478947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:58,951 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:16:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:58,971 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/26d02481005840eb982945f6fd3bd6d0 2024-11-17T03:16:58,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/fd297e57bc0c4ba3bc11bdd6743ce8ff is 50, key is test_row_0/C:col10/1731813416777/Put/seqid=0 2024-11-17T03:16:58,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742339_1515 (size=12001) 2024-11-17T03:16:59,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813479047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813479047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813479057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813479057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813479257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813479257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813479267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813479267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,389 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/fd297e57bc0c4ba3bc11bdd6743ce8ff 2024-11-17T03:16:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9b0f0b4f36741c0bf093dd2acbf5355 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355 2024-11-17T03:16:59,395 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355, entries=150, sequenceid=49, filesize=30.2 K 2024-11-17T03:16:59,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/26d02481005840eb982945f6fd3bd6d0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26d02481005840eb982945f6fd3bd6d0 2024-11-17T03:16:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,398 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26d02481005840eb982945f6fd3bd6d0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T03:16:59,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/fd297e57bc0c4ba3bc11bdd6743ce8ff as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fd297e57bc0c4ba3bc11bdd6743ce8ff 2024-11-17T03:16:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,406 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fd297e57bc0c4ba3bc11bdd6743ce8ff, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T03:16:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,406 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 97cf29522eb018ba919d51f502de21d3 in 1282ms, sequenceid=49, compaction requested=false 2024-11-17T03:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:16:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-17T03:16:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-17T03:16:59,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-17T03:16:59,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6570 sec 2024-11-17T03:16:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6630 sec 2024-11-17T03:16:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T03:16:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:16:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111708de94844bda41029cc20cfac465a936_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813419573/Put/seqid=0 2024-11-17T03:16:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742340_1516 (size=12154) 2024-11-17T03:16:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:16:59,596 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111708de94844bda41029cc20cfac465a936_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111708de94844bda41029cc20cfac465a936_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:16:59,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9e2a796f99346b09b54abfa2c8b6e7b, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:16:59,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9e2a796f99346b09b54abfa2c8b6e7b is 175, key is test_row_0/A:col10/1731813419573/Put/seqid=0 2024-11-17T03:16:59,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742341_1517 (size=30955) 2024-11-17T03:16:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813479610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813479609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813479610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813479612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813479727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813479727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813479727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813479727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813479936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813479936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813479937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:16:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813479937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:16:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,014 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9e2a796f99346b09b54abfa2c8b6e7b 2024-11-17T03:17:00,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/1bb82fd5bf9b40039ad53cf73cead1e5 is 50, key is test_row_0/B:col10/1731813419573/Put/seqid=0 2024-11-17T03:17:00,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742342_1518 (size=12001) 2024-11-17T03:17:00,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813480247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813480247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813480247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813480247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/1bb82fd5bf9b40039ad53cf73cead1e5 2024-11-17T03:17:00,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/7d92841fe59a488580308b3c20492f18 is 50, key is test_row_0/C:col10/1731813419573/Put/seqid=0 2024-11-17T03:17:00,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742343_1519 (size=12001) 2024-11-17T03:17:00,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:00,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/7d92841fe59a488580308b3c20492f18 2024-11-17T03:17:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/e9e2a796f99346b09b54abfa2c8b6e7b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b 2024-11-17T03:17:00,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b, entries=150, sequenceid=61, filesize=30.2 K 2024-11-17T03:17:00,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/1bb82fd5bf9b40039ad53cf73cead1e5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/1bb82fd5bf9b40039ad53cf73cead1e5 2024-11-17T03:17:00,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/1bb82fd5bf9b40039ad53cf73cead1e5, entries=150, sequenceid=61, filesize=11.7 K 2024-11-17T03:17:00,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/7d92841fe59a488580308b3c20492f18 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7d92841fe59a488580308b3c20492f18 2024-11-17T03:17:00,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7d92841fe59a488580308b3c20492f18, entries=150, sequenceid=61, filesize=11.7 K 2024-11-17T03:17:00,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 97cf29522eb018ba919d51f502de21d3 in 1269ms, sequenceid=61, compaction requested=true 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:17:00,843 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:00,843 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:17:00,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/B is initiating minor compaction (all files) 2024-11-17T03:17:00,844 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/B in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:00,844 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59fdc51486a34f38bb2618fbcd81b4f2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26d02481005840eb982945f6fd3bd6d0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/1bb82fd5bf9b40039ad53cf73cead1e5] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=35.2 K 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/A is initiating minor compaction (all files) 2024-11-17T03:17:00,844 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/A in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:00,844 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=90.7 K 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 59fdc51486a34f38bb2618fbcd81b4f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813416738 2024-11-17T03:17:00,844 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b] 2024-11-17T03:17:00,844 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d02481005840eb982945f6fd3bd6d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731813416775 2024-11-17T03:17:00,845 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b00e60e291a44758ab0bd14d69ce843, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813416738 2024-11-17T03:17:00,845 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9b0f0b4f36741c0bf093dd2acbf5355, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731813416775 2024-11-17T03:17:00,845 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb82fd5bf9b40039ad53cf73cead1e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731813418939 2024-11-17T03:17:00,846 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9e2a796f99346b09b54abfa2c8b6e7b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731813418939 2024-11-17T03:17:00,850 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:00,851 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:00,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/5496daa73af341cba63a101f830fd134 is 50, key is test_row_0/B:col10/1731813419573/Put/seqid=0 2024-11-17T03:17:00,853 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111774bf0373d92e4c6eaa477b8019fb709b_97cf29522eb018ba919d51f502de21d3 store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:00,855 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111774bf0373d92e4c6eaa477b8019fb709b_97cf29522eb018ba919d51f502de21d3, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:00,855 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111774bf0373d92e4c6eaa477b8019fb709b_97cf29522eb018ba919d51f502de21d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:00,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742344_1520 (size=12104) 2024-11-17T03:17:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742345_1521 (size=4469) 2024-11-17T03:17:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T03:17:00,917 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:00,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-17T03:17:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:00,921 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:17:00,922 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:17:00,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:17:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:00,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T03:17:00,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:00,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:17:00,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:00,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:17:00,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:00,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T03:17:00,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f4831f90800e48dfae3faca25a1251b1_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813419609/Put/seqid=0 2024-11-17T03:17:00,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742346_1522 (size=12154) 2024-11-17T03:17:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813481014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:01,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813481127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:01,265 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/5496daa73af341cba63a101f830fd134 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/5496daa73af341cba63a101f830fd134 2024-11-17T03:17:01,268 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/B of 97cf29522eb018ba919d51f502de21d3 into 5496daa73af341cba63a101f830fd134(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:01,269 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:01,269 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/B, priority=13, startTime=1731813420843; duration=0sec 2024-11-17T03:17:01,269 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:01,269 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:B 2024-11-17T03:17:01,269 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:17:01,269 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:17:01,269 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/C is initiating minor compaction (all files) 2024-11-17T03:17:01,269 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/C in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,270 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/780d37160043418e90ed5c257ce172c0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fd297e57bc0c4ba3bc11bdd6743ce8ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7d92841fe59a488580308b3c20492f18] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=35.2 K 2024-11-17T03:17:01,270 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 780d37160043418e90ed5c257ce172c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731813416738 2024-11-17T03:17:01,270 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting fd297e57bc0c4ba3bc11bdd6743ce8ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731813416775 2024-11-17T03:17:01,271 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d92841fe59a488580308b3c20492f18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731813418939 2024-11-17T03:17:01,273 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#A#compaction#442 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:01,273 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/42acea342955405d8b52e01361302704 is 175, key is test_row_0/A:col10/1731813419573/Put/seqid=0 2024-11-17T03:17:01,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742347_1523 (size=31058) 2024-11-17T03:17:01,277 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:01,278 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/9a41af74a92c47318b897e065cdd61e8 is 50, key is test_row_0/C:col10/1731813419573/Put/seqid=0 2024-11-17T03:17:01,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742348_1524 (size=12104) 2024-11-17T03:17:01,282 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/42acea342955405d8b52e01361302704 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/42acea342955405d8b52e01361302704 2024-11-17T03:17:01,285 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/9a41af74a92c47318b897e065cdd61e8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/9a41af74a92c47318b897e065cdd61e8 2024-11-17T03:17:01,286 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/A of 97cf29522eb018ba919d51f502de21d3 into 42acea342955405d8b52e01361302704(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:01,286 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:01,286 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/A, priority=13, startTime=1731813420843; duration=0sec 2024-11-17T03:17:01,286 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:01,286 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:A 2024-11-17T03:17:01,289 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/C of 97cf29522eb018ba919d51f502de21d3 into 9a41af74a92c47318b897e065cdd61e8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:01,289 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:01,289 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/C, priority=13, startTime=1731813420843; duration=0sec 2024-11-17T03:17:01,290 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:01,290 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:C 2024-11-17T03:17:01,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813481337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:01,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f4831f90800e48dfae3faca25a1251b1_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f4831f90800e48dfae3faca25a1251b1_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:01,394 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b098d2b4bf764065a5ec296ee611e7d8, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:01,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b098d2b4bf764065a5ec296ee611e7d8 is 175, key is test_row_0/A:col10/1731813419609/Put/seqid=0 2024-11-17T03:17:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742349_1525 (size=30955) 2024-11-17T03:17:01,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813481646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813481767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813481767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,769 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813481768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,769 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,770 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_1' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813481777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:01,779 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:01,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b098d2b4bf764065a5ec296ee611e7d8 2024-11-17T03:17:01,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3113ca174f9f4ade81afae090cdf29da is 50, key is test_row_0/B:col10/1731813419609/Put/seqid=0 2024-11-17T03:17:01,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742350_1526 (size=12001) 2024-11-17T03:17:01,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:01,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:02,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:02,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:02,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813482157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:02,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:02,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3113ca174f9f4ade81afae090cdf29da 2024-11-17T03:17:02,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/b2843dcf567141569635ddf6c89834bb is 50, key is test_row_0/C:col10/1731813419609/Put/seqid=0 2024-11-17T03:17:02,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742351_1527 (size=12001) 2024-11-17T03:17:02,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:02,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:02,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:02,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:02,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:02,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/b2843dcf567141569635ddf6c89834bb 2024-11-17T03:17:02,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b098d2b4bf764065a5ec296ee611e7d8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8 2024-11-17T03:17:02,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8, entries=150, sequenceid=82, filesize=30.2 K 2024-11-17T03:17:02,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3113ca174f9f4ade81afae090cdf29da as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3113ca174f9f4ade81afae090cdf29da 2024-11-17T03:17:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3113ca174f9f4ade81afae090cdf29da, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T03:17:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/b2843dcf567141569635ddf6c89834bb as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/b2843dcf567141569635ddf6c89834bb 2024-11-17T03:17:02,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/b2843dcf567141569635ddf6c89834bb, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T03:17:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 97cf29522eb018ba919d51f502de21d3 in 1655ms, sequenceid=82, compaction requested=false 2024-11-17T03:17:02,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T03:17:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,753 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T03:17:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f651a3572c3a4be89f4ebcfb1a6982fc_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_1/A:col10/1731813421004/Put/seqid=0 2024-11-17T03:17:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742352_1528 (size=9714) 2024-11-17T03:17:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,765 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f651a3572c3a4be89f4ebcfb1a6982fc_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f651a3572c3a4be89f4ebcfb1a6982fc_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b136bf18ff0640529f02112c257e649b, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b136bf18ff0640529f02112c257e649b is 175, key is test_row_1/A:col10/1731813421004/Put/seqid=0 2024-11-17T03:17:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742353_1529 (size=22361) 2024-11-17T03:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,770 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b136bf18ff0640529f02112c257e649b 2024-11-17T03:17:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3a6284c5837e42ac92184d74a8a843b2 is 50, key is test_row_1/B:col10/1731813421004/Put/seqid=0 2024-11-17T03:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742354_1530 (size=9657) 2024-11-17T03:17:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,782 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3a6284c5837e42ac92184d74a8a843b2 2024-11-17T03:17:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/7735722f0fee41e2aac1daf5e1bd110a is 50, key is test_row_1/C:col10/1731813421004/Put/seqid=0 2024-11-17T03:17:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742355_1531 (size=9657) 2024-11-17T03:17:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:03,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,192 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/7735722f0fee41e2aac1daf5e1bd110a 2024-11-17T03:17:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b136bf18ff0640529f02112c257e649b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b 2024-11-17T03:17:03,201 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b, entries=100, sequenceid=96, filesize=21.8 K 2024-11-17T03:17:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3a6284c5837e42ac92184d74a8a843b2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3a6284c5837e42ac92184d74a8a843b2 2024-11-17T03:17:03,208 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3a6284c5837e42ac92184d74a8a843b2, entries=100, sequenceid=96, filesize=9.4 K 2024-11-17T03:17:03,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/7735722f0fee41e2aac1daf5e1bd110a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7735722f0fee41e2aac1daf5e1bd110a 2024-11-17T03:17:03,212 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7735722f0fee41e2aac1daf5e1bd110a, entries=100, sequenceid=96, filesize=9.4 K 2024-11-17T03:17:03,212 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for 97cf29522eb018ba919d51f502de21d3 in 460ms, sequenceid=96, compaction requested=true 2024-11-17T03:17:03,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:03,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:03,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-17T03:17:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-17T03:17:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:03,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:17:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:03,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T03:17:03,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2930 sec 2024-11-17T03:17:03,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.3020 sec 2024-11-17T03:17:03,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ed70ff775c244aebb70c2673ce5a1ce3_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742356_1532 (size=12154) 2024-11-17T03:17:03,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:03,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ed70ff775c244aebb70c2673ce5a1ce3_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ed70ff775c244aebb70c2673ce5a1ce3_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:03,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/15bf0bddf2f54a42b1f765d049e087a9, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:03,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/15bf0bddf2f54a42b1f765d049e087a9 is 175, key is test_row_0/A:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:03,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742357_1533 (size=30955) 2024-11-17T03:17:03,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/15bf0bddf2f54a42b1f765d049e087a9 2024-11-17T03:17:03,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/6b1eb8253d314c5fbd84a4c066e12b0d is 50, key is test_row_0/B:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:03,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742358_1534 (size=12001) 2024-11-17T03:17:03,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/6b1eb8253d314c5fbd84a4c066e12b0d 2024-11-17T03:17:03,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813483311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/db460886786747948e74979aad1af25b is 50, key is test_row_0/C:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:03,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742359_1535 (size=12001) 2024-11-17T03:17:03,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813483426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813483637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/db460886786747948e74979aad1af25b 2024-11-17T03:17:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/15bf0bddf2f54a42b1f765d049e087a9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9 2024-11-17T03:17:03,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9, entries=150, sequenceid=110, filesize=30.2 K 2024-11-17T03:17:03,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/6b1eb8253d314c5fbd84a4c066e12b0d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6b1eb8253d314c5fbd84a4c066e12b0d 2024-11-17T03:17:03,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6b1eb8253d314c5fbd84a4c066e12b0d, entries=150, sequenceid=110, filesize=11.7 K 2024-11-17T03:17:03,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/db460886786747948e74979aad1af25b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/db460886786747948e74979aad1af25b 2024-11-17T03:17:03,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/db460886786747948e74979aad1af25b, entries=150, sequenceid=110, filesize=11.7 K 2024-11-17T03:17:03,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 97cf29522eb018ba919d51f502de21d3 in 530ms, sequenceid=110, compaction requested=true 2024-11-17T03:17:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:17:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:03,747 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:17:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:17:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:03,747 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:17:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:17:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:03,747 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 115329 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:17:03,747 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:17:03,747 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/B is initiating minor compaction (all files) 2024-11-17T03:17:03,747 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/A is initiating minor compaction (all files) 2024-11-17T03:17:03,748 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/A in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:03,748 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/B in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:03,748 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/5496daa73af341cba63a101f830fd134, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3113ca174f9f4ade81afae090cdf29da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3a6284c5837e42ac92184d74a8a843b2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6b1eb8253d314c5fbd84a4c066e12b0d] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=44.7 K 2024-11-17T03:17:03,748 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/42acea342955405d8b52e01361302704, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=112.6 K 2024-11-17T03:17:03,748 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/42acea342955405d8b52e01361302704, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9] 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 5496daa73af341cba63a101f830fd134, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731813418939 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42acea342955405d8b52e01361302704, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731813418939 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3113ca174f9f4ade81afae090cdf29da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731813419601 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b098d2b4bf764065a5ec296ee611e7d8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731813419601 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6284c5837e42ac92184d74a8a843b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813421004 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b136bf18ff0640529f02112c257e649b, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813421004 2024-11-17T03:17:03,748 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1eb8253d314c5fbd84a4c066e12b0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731813423198 2024-11-17T03:17:03,749 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15bf0bddf2f54a42b1f765d049e087a9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731813423198 2024-11-17T03:17:03,753 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:03,755 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:03,756 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/7ac942427fbb4f09840da0fde1c31ed5 is 50, key is test_row_0/B:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:03,757 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111700b258b944d943e48b9d8371956f0043_97cf29522eb018ba919d51f502de21d3 store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:03,760 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111700b258b944d943e48b9d8371956f0043_97cf29522eb018ba919d51f502de21d3, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:03,760 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111700b258b944d943e48b9d8371956f0043_97cf29522eb018ba919d51f502de21d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:03,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742360_1536 (size=12241) 2024-11-17T03:17:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742361_1537 (size=4469) 2024-11-17T03:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:03,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T03:17:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:03,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e2cd212a569441b28af24a4513ba43b0_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813423295/Put/seqid=0 2024-11-17T03:17:03,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742362_1538 (size=14644) 2024-11-17T03:17:03,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813483799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813483841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813483841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813483841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813483947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813483947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813483957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813483957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813483957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813484157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,169 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/7ac942427fbb4f09840da0fde1c31ed5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/7ac942427fbb4f09840da0fde1c31ed5 2024-11-17T03:17:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813484167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813484167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813484167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,175 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#A#compaction#453 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:04,175 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/B of 97cf29522eb018ba919d51f502de21d3 into 7ac942427fbb4f09840da0fde1c31ed5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:04,175 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:04,175 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/B, priority=12, startTime=1731813423747; duration=0sec 2024-11-17T03:17:04,175 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:04,175 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:B 2024-11-17T03:17:04,175 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T03:17:04,175 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b7945c51b8a64f6e8870f3aa5b7025ae is 175, key is test_row_0/A:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:04,177 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T03:17:04,177 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/C is initiating minor compaction (all files) 2024-11-17T03:17:04,177 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/C in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:04,177 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/9a41af74a92c47318b897e065cdd61e8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/b2843dcf567141569635ddf6c89834bb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7735722f0fee41e2aac1daf5e1bd110a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/db460886786747948e74979aad1af25b] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=44.7 K 2024-11-17T03:17:04,177 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a41af74a92c47318b897e065cdd61e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731813418939 2024-11-17T03:17:04,178 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting b2843dcf567141569635ddf6c89834bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731813419601 2024-11-17T03:17:04,178 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7735722f0fee41e2aac1daf5e1bd110a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731813421004 2024-11-17T03:17:04,180 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting db460886786747948e74979aad1af25b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731813423198 2024-11-17T03:17:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742363_1539 (size=31195) 2024-11-17T03:17:04,185 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b7945c51b8a64f6e8870f3aa5b7025ae as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b7945c51b8a64f6e8870f3aa5b7025ae 2024-11-17T03:17:04,190 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/A of 97cf29522eb018ba919d51f502de21d3 into b7945c51b8a64f6e8870f3aa5b7025ae(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:04,190 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:04,190 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/A, priority=12, startTime=1731813423747; duration=0sec 2024-11-17T03:17:04,190 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:04,190 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:A 2024-11-17T03:17:04,194 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:04,195 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/afbb4b22da7b4e2190f16835265a85b0 is 50, key is test_row_0/C:col10/1731813423198/Put/seqid=0 2024-11-17T03:17:04,196 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:04,201 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e2cd212a569441b28af24a4513ba43b0_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e2cd212a569441b28af24a4513ba43b0_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:04,202 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/2422eab2a78544fa9970127a8bd5f85e, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:04,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/2422eab2a78544fa9970127a8bd5f85e is 175, key is test_row_0/A:col10/1731813423295/Put/seqid=0 2024-11-17T03:17:04,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742364_1540 (size=12241) 2024-11-17T03:17:04,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742365_1541 (size=39599) 2024-11-17T03:17:04,231 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/2422eab2a78544fa9970127a8bd5f85e 2024-11-17T03:17:04,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/26652ee9a36a41668ff59285073f3b92 is 50, key is test_row_0/B:col10/1731813423295/Put/seqid=0 2024-11-17T03:17:04,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742366_1542 (size=12051) 2024-11-17T03:17:04,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/26652ee9a36a41668ff59285073f3b92 2024-11-17T03:17:04,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/4a52e07e556344ff94716eb7fb9be4f1 is 50, key is test_row_0/C:col10/1731813423295/Put/seqid=0 2024-11-17T03:17:04,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742367_1543 (size=12051) 2024-11-17T03:17:04,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/4a52e07e556344ff94716eb7fb9be4f1 2024-11-17T03:17:04,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/2422eab2a78544fa9970127a8bd5f85e as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e 2024-11-17T03:17:04,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e, entries=200, sequenceid=129, filesize=38.7 K 2024-11-17T03:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/26652ee9a36a41668ff59285073f3b92 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26652ee9a36a41668ff59285073f3b92 2024-11-17T03:17:04,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26652ee9a36a41668ff59285073f3b92, entries=150, sequenceid=129, filesize=11.8 K 2024-11-17T03:17:04,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/4a52e07e556344ff94716eb7fb9be4f1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/4a52e07e556344ff94716eb7fb9be4f1 2024-11-17T03:17:04,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/4a52e07e556344ff94716eb7fb9be4f1, entries=150, sequenceid=129, filesize=11.8 K 2024-11-17T03:17:04,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 97cf29522eb018ba919d51f502de21d3 in 518ms, sequenceid=129, compaction requested=false 2024-11-17T03:17:04,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:04,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:17:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:04,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:04,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174da056788c8045a0837129717754118f_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:04,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742368_1544 (size=12304) 2024-11-17T03:17:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813484529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813484529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813484529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813484531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813484533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,610 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/afbb4b22da7b4e2190f16835265a85b0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/afbb4b22da7b4e2190f16835265a85b0 2024-11-17T03:17:04,615 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/C of 97cf29522eb018ba919d51f502de21d3 into afbb4b22da7b4e2190f16835265a85b0(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:04,615 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:04,615 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/C, priority=12, startTime=1731813423747; duration=0sec 2024-11-17T03:17:04,616 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:04,616 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:C 2024-11-17T03:17:04,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813484647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813484647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813484647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813484648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813484648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813484857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813484857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813484857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813484858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813484858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:04,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:04,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174da056788c8045a0837129717754118f_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174da056788c8045a0837129717754118f_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:04,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/d298a522264d4334b75e0e7e0698da8d, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:04,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/d298a522264d4334b75e0e7e0698da8d is 175, key is test_row_0/A:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:04,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742369_1545 (size=31105) 2024-11-17T03:17:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T03:17:05,088 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:05,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:17:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-17T03:17:05,090 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:17:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T03:17:05,090 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:17:05,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:17:05,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813485167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T03:17:05,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T03:17:05,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:05,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:05,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:05,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/d298a522264d4334b75e0e7e0698da8d 2024-11-17T03:17:05,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/65d40bbfceac4f289b023f0af8e62c90 is 50, key is test_row_0/B:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:05,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T03:17:05,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:05,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:05,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T03:17:05,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742370_1546 (size=12151) 2024-11-17T03:17:05,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/65d40bbfceac4f289b023f0af8e62c90 2024-11-17T03:17:05,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/6eb7c4ccac73490a8f2500deb3b0c850 is 50, key is test_row_0/C:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:05,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742371_1547 (size=12151) 2024-11-17T03:17:05,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/6eb7c4ccac73490a8f2500deb3b0c850 2024-11-17T03:17:05,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/d298a522264d4334b75e0e7e0698da8d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d 2024-11-17T03:17:05,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d, entries=150, sequenceid=146, filesize=30.4 K 2024-11-17T03:17:05,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/65d40bbfceac4f289b023f0af8e62c90 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/65d40bbfceac4f289b023f0af8e62c90 2024-11-17T03:17:05,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/65d40bbfceac4f289b023f0af8e62c90, entries=150, sequenceid=146, filesize=11.9 K 2024-11-17T03:17:05,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/6eb7c4ccac73490a8f2500deb3b0c850 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/6eb7c4ccac73490a8f2500deb3b0c850 2024-11-17T03:17:05,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/6eb7c4ccac73490a8f2500deb3b0c850, entries=150, sequenceid=146, filesize=11.9 K 2024-11-17T03:17:05,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 97cf29522eb018ba919d51f502de21d3 in 1004ms, sequenceid=146, compaction requested=true 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:17:05,466 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:17:05,466 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:17:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:05,469 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:17:05,469 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/B is initiating minor compaction (all files) 2024-11-17T03:17:05,469 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/B in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,469 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/7ac942427fbb4f09840da0fde1c31ed5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26652ee9a36a41668ff59285073f3b92, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/65d40bbfceac4f289b023f0af8e62c90] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=35.6 K 2024-11-17T03:17:05,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:17:05,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/A is initiating minor compaction (all files) 2024-11-17T03:17:05,470 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/A in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,470 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b7945c51b8a64f6e8870f3aa5b7025ae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=99.5 K 2024-11-17T03:17:05,470 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b7945c51b8a64f6e8870f3aa5b7025ae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d] 2024-11-17T03:17:05,470 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7945c51b8a64f6e8870f3aa5b7025ae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731813423198 2024-11-17T03:17:05,470 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac942427fbb4f09840da0fde1c31ed5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731813423198 2024-11-17T03:17:05,471 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2422eab2a78544fa9970127a8bd5f85e, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813423287 2024-11-17T03:17:05,471 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 26652ee9a36a41668ff59285073f3b92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813423287 2024-11-17T03:17:05,471 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d40bbfceac4f289b023f0af8e62c90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731813423802 2024-11-17T03:17:05,471 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting d298a522264d4334b75e0e7e0698da8d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731813423802 2024-11-17T03:17:05,485 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:05,499 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#B#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:05,499 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/328cf1b2ef4f4382b961ee4e1a3a64b5 is 50, key is test_row_0/B:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:05,499 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411178b1ec7d156744da6b1fd09e90c90bc2c_97cf29522eb018ba919d51f502de21d3 store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:05,501 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411178b1ec7d156744da6b1fd09e90c90bc2c_97cf29522eb018ba919d51f502de21d3, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:05,502 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178b1ec7d156744da6b1fd09e90c90bc2c_97cf29522eb018ba919d51f502de21d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:05,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742373_1549 (size=4469) 2024-11-17T03:17:05,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742372_1548 (size=12493) 2024-11-17T03:17:05,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,551 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e46d1adad1fd42eabdd0439c53fc9805_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813424518/Put/seqid=0 2024-11-17T03:17:05,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742374_1550 (size=12304) 2024-11-17T03:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:05,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:05,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813485698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813485698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813485707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813485708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813485708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T03:17:05,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813485817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813485818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813485818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813485818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813485827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:05,931 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#A#compaction#462 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:05,932 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/1641985a8cb04503a9237dcff3cdb85d is 175, key is test_row_0/A:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:05,951 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/328cf1b2ef4f4382b961ee4e1a3a64b5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/328cf1b2ef4f4382b961ee4e1a3a64b5 2024-11-17T03:17:05,960 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/B of 97cf29522eb018ba919d51f502de21d3 into 328cf1b2ef4f4382b961ee4e1a3a64b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:05,960 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:05,960 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/B, priority=13, startTime=1731813425466; duration=0sec 2024-11-17T03:17:05,960 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:05,960 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:B 2024-11-17T03:17:05,960 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T03:17:05,961 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T03:17:05,961 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/C is initiating minor compaction (all files) 2024-11-17T03:17:05,961 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/C in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:05,961 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/afbb4b22da7b4e2190f16835265a85b0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/4a52e07e556344ff94716eb7fb9be4f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/6eb7c4ccac73490a8f2500deb3b0c850] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=35.6 K 2024-11-17T03:17:05,962 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting afbb4b22da7b4e2190f16835265a85b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731813423198 2024-11-17T03:17:05,962 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a52e07e556344ff94716eb7fb9be4f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731813423287 2024-11-17T03:17:05,962 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb7c4ccac73490a8f2500deb3b0c850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731813423802 2024-11-17T03:17:05,974 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:05,974 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/516d0ae050bd414e89f7085abfb8bd0b is 50, key is test_row_0/C:col10/1731813423802/Put/seqid=0 2024-11-17T03:17:05,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742375_1551 (size=31447) 2024-11-17T03:17:05,982 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/1641985a8cb04503a9237dcff3cdb85d as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1641985a8cb04503a9237dcff3cdb85d 2024-11-17T03:17:05,987 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/A of 97cf29522eb018ba919d51f502de21d3 into 1641985a8cb04503a9237dcff3cdb85d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:05,987 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:05,987 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/A, priority=13, startTime=1731813425466; duration=0sec 2024-11-17T03:17:05,988 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:05,988 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:A 2024-11-17T03:17:05,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:05,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742376_1552 (size=12493) 2024-11-17T03:17:06,009 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e46d1adad1fd42eabdd0439c53fc9805_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e46d1adad1fd42eabdd0439c53fc9805_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:06,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b80ff718239e4d5487ed9a75a36c0bb5, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b80ff718239e4d5487ed9a75a36c0bb5 is 175, key is test_row_0/A:col10/1731813424518/Put/seqid=0 2024-11-17T03:17:06,015 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/516d0ae050bd414e89f7085abfb8bd0b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/516d0ae050bd414e89f7085abfb8bd0b 2024-11-17T03:17:06,019 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/C of 97cf29522eb018ba919d51f502de21d3 into 516d0ae050bd414e89f7085abfb8bd0b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:06,019 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:06,019 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/C, priority=13, startTime=1731813425466; duration=0sec 2024-11-17T03:17:06,019 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:06,019 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:C 2024-11-17T03:17:06,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742377_1553 (size=31105) 2024-11-17T03:17:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813486027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813486027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813486027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813486027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813486036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T03:17:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813486337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813486337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813486337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813486337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813486347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,426 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b80ff718239e4d5487ed9a75a36c0bb5 2024-11-17T03:17:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/59bd637bcc364f2c9289620be7396e93 is 50, key is test_row_0/B:col10/1731813424518/Put/seqid=0 2024-11-17T03:17:06,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742378_1554 (size=12151) 2024-11-17T03:17:06,454 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/59bd637bcc364f2c9289620be7396e93 2024-11-17T03:17:06,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/fa440fe41227404cbe6e0a3f5ea3f224 is 50, key is test_row_0/C:col10/1731813424518/Put/seqid=0 2024-11-17T03:17:06,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742379_1555 (size=12151) 2024-11-17T03:17:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813486857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813486857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:06,885 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/fa440fe41227404cbe6e0a3f5ea3f224 2024-11-17T03:17:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/b80ff718239e4d5487ed9a75a36c0bb5 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5 2024-11-17T03:17:06,890 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5, entries=150, sequenceid=164, filesize=30.4 K 2024-11-17T03:17:06,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/59bd637bcc364f2c9289620be7396e93 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59bd637bcc364f2c9289620be7396e93 2024-11-17T03:17:06,893 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59bd637bcc364f2c9289620be7396e93, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T03:17:06,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/fa440fe41227404cbe6e0a3f5ea3f224 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fa440fe41227404cbe6e0a3f5ea3f224 2024-11-17T03:17:06,896 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fa440fe41227404cbe6e0a3f5ea3f224, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T03:17:06,896 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 97cf29522eb018ba919d51f502de21d3 in 1345ms, sequenceid=164, compaction requested=false 2024-11-17T03:17:06,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:06,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:06,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-17T03:17:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-17T03:17:06,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-17T03:17:06,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-11-17T03:17:06,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8110 sec 2024-11-17T03:17:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T03:17:07,237 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:07,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:17:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T03:17:07,239 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:17:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T03:17:07,240 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:17:07,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:17:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T03:17:07,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T03:17:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:07,392 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T03:17:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e78e600d3cd44538cdff2a2c5dd1e86_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813425706/Put/seqid=0 2024-11-17T03:17:07,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742380_1556 (size=12304) 2024-11-17T03:17:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:07,404 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e78e600d3cd44538cdff2a2c5dd1e86_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e78e600d3cd44538cdff2a2c5dd1e86_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:07,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/dfbc13f8da244e1db6a27eb361842a96, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:07,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/dfbc13f8da244e1db6a27eb361842a96 is 175, key is test_row_0/A:col10/1731813425706/Put/seqid=0 2024-11-17T03:17:07,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742381_1557 (size=31105) 2024-11-17T03:17:07,408 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/dfbc13f8da244e1db6a27eb361842a96 2024-11-17T03:17:07,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/837babc4dd504ddb8bb3fc9a4d7696c4 is 50, key is test_row_0/B:col10/1731813425706/Put/seqid=0 2024-11-17T03:17:07,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742382_1558 (size=12151) 2024-11-17T03:17:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T03:17:07,817 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/837babc4dd504ddb8bb3fc9a4d7696c4 2024-11-17T03:17:07,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/c930fa71db0d445eadc8febd0716196a is 50, key is test_row_0/C:col10/1731813425706/Put/seqid=0 2024-11-17T03:17:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:07,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:07,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742383_1559 (size=12151) 2024-11-17T03:17:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T03:17:07,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813487902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813487904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:07,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813487905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:07,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813487908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813487908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813488018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813488018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813488018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813488018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813488018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813488226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813488227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813488227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813488227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813488227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,262 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/c930fa71db0d445eadc8febd0716196a 2024-11-17T03:17:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/dfbc13f8da244e1db6a27eb361842a96 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96 2024-11-17T03:17:08,268 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96, entries=150, sequenceid=183, filesize=30.4 K 2024-11-17T03:17:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/837babc4dd504ddb8bb3fc9a4d7696c4 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/837babc4dd504ddb8bb3fc9a4d7696c4 2024-11-17T03:17:08,271 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/837babc4dd504ddb8bb3fc9a4d7696c4, entries=150, sequenceid=183, filesize=11.9 K 2024-11-17T03:17:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/c930fa71db0d445eadc8febd0716196a as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/c930fa71db0d445eadc8febd0716196a 2024-11-17T03:17:08,274 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/c930fa71db0d445eadc8febd0716196a, entries=150, sequenceid=183, filesize=11.9 K 2024-11-17T03:17:08,274 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 97cf29522eb018ba919d51f502de21d3 in 882ms, sequenceid=183, compaction requested=true 2024-11-17T03:17:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:08,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T03:17:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T03:17:08,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T03:17:08,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-17T03:17:08,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0390 sec 2024-11-17T03:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T03:17:08,377 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:08,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:17:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T03:17:08,379 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:17:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T03:17:08,380 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:17:08,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:17:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T03:17:08,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T03:17:08,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:08,532 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T03:17:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e43d4c729cf4ebb92e870361a864ce1_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813427907/Put/seqid=0 2024-11-17T03:17:08,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742384_1560 (size=12304) 2024-11-17T03:17:08,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813488548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813488549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813488550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813488552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813488553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813488657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813488657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813488657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813488667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813488667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T03:17:08,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813488866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813488867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813488867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813488877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813488877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:08,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:08,947 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e43d4c729cf4ebb92e870361a864ce1_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e43d4c729cf4ebb92e870361a864ce1_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:08,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/bf6603d1bc5c4903ad97e0c8b1c35beb, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:08,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/bf6603d1bc5c4903ad97e0c8b1c35beb is 175, key is test_row_0/A:col10/1731813427907/Put/seqid=0 2024-11-17T03:17:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742385_1561 (size=31105) 2024-11-17T03:17:08,954 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/bf6603d1bc5c4903ad97e0c8b1c35beb 2024-11-17T03:17:08,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/feb9d3f5aba746cb9a35efe1f0d7357b is 50, key is test_row_0/B:col10/1731813427907/Put/seqid=0 2024-11-17T03:17:08,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742386_1562 (size=12151) 2024-11-17T03:17:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T03:17:09,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813489177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813489177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813489177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813489187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813489187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,365 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/feb9d3f5aba746cb9a35efe1f0d7357b 2024-11-17T03:17:09,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/3adb104739fb467886492db2043ab857 is 50, key is test_row_0/C:col10/1731813427907/Put/seqid=0 2024-11-17T03:17:09,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742387_1563 (size=12151) 2024-11-17T03:17:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T03:17:09,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813489687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813489687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813489687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813489697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813489697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:09,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:09,782 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/3adb104739fb467886492db2043ab857 2024-11-17T03:17:09,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/bf6603d1bc5c4903ad97e0c8b1c35beb as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb 2024-11-17T03:17:09,787 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb, entries=150, sequenceid=200, filesize=30.4 K 2024-11-17T03:17:09,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/feb9d3f5aba746cb9a35efe1f0d7357b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/feb9d3f5aba746cb9a35efe1f0d7357b 2024-11-17T03:17:09,791 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/feb9d3f5aba746cb9a35efe1f0d7357b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T03:17:09,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/3adb104739fb467886492db2043ab857 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/3adb104739fb467886492db2043ab857 2024-11-17T03:17:09,795 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/3adb104739fb467886492db2043ab857, entries=150, sequenceid=200, filesize=11.9 K 2024-11-17T03:17:09,796 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 97cf29522eb018ba919d51f502de21d3 in 1264ms, sequenceid=200, compaction requested=true 2024-11-17T03:17:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T03:17:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T03:17:09,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T03:17:09,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-17T03:17:09,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4210 sec 2024-11-17T03:17:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T03:17:10,527 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:10,528 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:17:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T03:17:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T03:17:10,529 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:17:10,530 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:17:10,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:17:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T03:17:10,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T03:17:10,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:10,682 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T03:17:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:10,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:10,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171718c5de53f244468185b5dde4e3d8b2_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813428552/Put/seqid=0 2024-11-17T03:17:10,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742388_1564 (size=12304) 2024-11-17T03:17:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:10,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:10,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813490717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,723 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813490719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813490721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813490722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813490723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813490827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813490837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813490837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813490837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:10,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T03:17:11,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813491037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813491047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813491047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813491047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:11,095 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171718c5de53f244468185b5dde4e3d8b2_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171718c5de53f244468185b5dde4e3d8b2_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/35032aa12d664a748ef4c5cd86daf6da, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:11,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/35032aa12d664a748ef4c5cd86daf6da is 175, key is test_row_0/A:col10/1731813428552/Put/seqid=0 2024-11-17T03:17:11,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742389_1565 (size=31105) 2024-11-17T03:17:11,112 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/35032aa12d664a748ef4c5cd86daf6da 2024-11-17T03:17:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3532e78765b14662b23c7b38546e0997 is 50, key is test_row_0/B:col10/1731813428552/Put/seqid=0 2024-11-17T03:17:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742390_1566 (size=12151) 2024-11-17T03:17:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T03:17:11,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813491347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813491357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813491357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813491357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,527 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3532e78765b14662b23c7b38546e0997 2024-11-17T03:17:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/38ae3abacf084499ab1ee61a6c1403d8 is 50, key is test_row_0/C:col10/1731813428552/Put/seqid=0 2024-11-17T03:17:11,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742391_1567 (size=12151) 2024-11-17T03:17:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T03:17:11,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813491857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813491867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813491867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813491867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:11,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:11,938 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/38ae3abacf084499ab1ee61a6c1403d8 2024-11-17T03:17:11,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/35032aa12d664a748ef4c5cd86daf6da as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da 2024-11-17T03:17:11,943 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da, entries=150, sequenceid=215, filesize=30.4 K 2024-11-17T03:17:11,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/3532e78765b14662b23c7b38546e0997 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3532e78765b14662b23c7b38546e0997 2024-11-17T03:17:11,947 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3532e78765b14662b23c7b38546e0997, entries=150, sequenceid=215, filesize=11.9 K 2024-11-17T03:17:11,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/38ae3abacf084499ab1ee61a6c1403d8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38ae3abacf084499ab1ee61a6c1403d8 2024-11-17T03:17:11,950 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38ae3abacf084499ab1ee61a6c1403d8, entries=150, sequenceid=215, filesize=11.9 K 2024-11-17T03:17:11,951 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 97cf29522eb018ba919d51f502de21d3 in 1269ms, sequenceid=215, compaction requested=true 2024-11-17T03:17:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:11,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T03:17:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T03:17:11,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T03:17:11,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4230 sec 2024-11-17T03:17:11,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4270 sec 2024-11-17T03:17:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T03:17:12,677 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:12,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T03:17:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T03:17:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:12,679 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T03:17:12,679 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T03:17:12,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T03:17:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:12,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T03:17:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:12,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d0376879fc744216bc5a1dafdf53216f_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:12,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742392_1568 (size=14794) 2024-11-17T03:17:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:12,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813492807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:12,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:12,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:12,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:12,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813492866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:12,869 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813492876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:12,880 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_0' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:12,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813492887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:12,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813492887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:12,891 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,891 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:12,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813492917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:12,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:12,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:12,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:12,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:12,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:12,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:13,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813493127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:13,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:13,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:13,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d0376879fc744216bc5a1dafdf53216f_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d0376879fc744216bc5a1dafdf53216f_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:13,161 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/11f8f419621449b3a48715e33de9a39b, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:13,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/11f8f419621449b3a48715e33de9a39b is 175, key is test_row_0/A:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742393_1569 (size=39749) 2024-11-17T03:17:13,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:13,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813493437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:13,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:13,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/11f8f419621449b3a48715e33de9a39b 2024-11-17T03:17:13,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/6ba9d3388f3d461583eb9b40c187e7cd is 50, key is test_row_0/B:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:13,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742394_1570 (size=12151) 2024-11-17T03:17:13,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:13,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:13,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:13,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813493957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:13,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/6ba9d3388f3d461583eb9b40c187e7cd 2024-11-17T03:17:13,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/38c272bbce8746e6b33296693f98abd0 is 50, key is test_row_0/C:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:14,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742395_1571 (size=12151) 2024-11-17T03:17:14,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:14,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:14,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T03:17:14,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/38c272bbce8746e6b33296693f98abd0 2024-11-17T03:17:14,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/11f8f419621449b3a48715e33de9a39b as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b 2024-11-17T03:17:14,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b, entries=200, sequenceid=233, filesize=38.8 K 2024-11-17T03:17:14,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/6ba9d3388f3d461583eb9b40c187e7cd as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6ba9d3388f3d461583eb9b40c187e7cd 2024-11-17T03:17:14,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6ba9d3388f3d461583eb9b40c187e7cd, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T03:17:14,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/38c272bbce8746e6b33296693f98abd0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38c272bbce8746e6b33296693f98abd0 2024-11-17T03:17:14,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38c272bbce8746e6b33296693f98abd0, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T03:17:14,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 97cf29522eb018ba919d51f502de21d3 in 1668ms, sequenceid=233, compaction requested=true 2024-11-17T03:17:14,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:17:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:14,419 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T03:17:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:17:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:14,419 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T03:17:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T03:17:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:14,420 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195616 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T03:17:14,420 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/A is initiating minor compaction (all files) 2024-11-17T03:17:14,420 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/A in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,420 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1641985a8cb04503a9237dcff3cdb85d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=191.0 K 2024-11-17T03:17:14,420 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,420 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. files: [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1641985a8cb04503a9237dcff3cdb85d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b] 2024-11-17T03:17:14,420 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73248 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T03:17:14,420 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/B is initiating minor compaction (all files) 2024-11-17T03:17:14,420 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/B in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,420 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/328cf1b2ef4f4382b961ee4e1a3a64b5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59bd637bcc364f2c9289620be7396e93, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/837babc4dd504ddb8bb3fc9a4d7696c4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/feb9d3f5aba746cb9a35efe1f0d7357b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3532e78765b14662b23c7b38546e0997, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6ba9d3388f3d461583eb9b40c187e7cd] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=71.5 K 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1641985a8cb04503a9237dcff3cdb85d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731813423802 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 328cf1b2ef4f4382b961ee4e1a3a64b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731813423802 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting b80ff718239e4d5487ed9a75a36c0bb5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731813424518 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bd637bcc364f2c9289620be7396e93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731813424518 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbc13f8da244e1db6a27eb361842a96, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731813425704 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 837babc4dd504ddb8bb3fc9a4d7696c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731813425704 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6603d1bc5c4903ad97e0c8b1c35beb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731813427869 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting feb9d3f5aba746cb9a35efe1f0d7357b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731813427869 2024-11-17T03:17:14,421 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35032aa12d664a748ef4c5cd86daf6da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731813428547 2024-11-17T03:17:14,422 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11f8f419621449b3a48715e33de9a39b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731813430718 2024-11-17T03:17:14,422 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3532e78765b14662b23c7b38546e0997, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731813428547 2024-11-17T03:17:14,422 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba9d3388f3d461583eb9b40c187e7cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731813430718 2024-11-17T03:17:14,432 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:14,434 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117988c4a29c8494700a90dc6e6c41750c9_97cf29522eb018ba919d51f502de21d3 store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:14,435 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#B#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:14,435 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/bdc57b31a5b843588ac0a6c4eddc4c66 is 50, key is test_row_0/B:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:14,437 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117988c4a29c8494700a90dc6e6c41750c9_97cf29522eb018ba919d51f502de21d3, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:14,437 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117988c4a29c8494700a90dc6e6c41750c9_97cf29522eb018ba919d51f502de21d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:14,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742396_1572 (size=12697) 2024-11-17T03:17:14,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742397_1573 (size=4469) 2024-11-17T03:17:14,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38753 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T03:17:14,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,512 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T03:17:14,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:14,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c2a1f9605e6d4d64b94523738d8dc33f_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813432794/Put/seqid=0 2024-11-17T03:17:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742398_1574 (size=12304) 2024-11-17T03:17:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:14,841 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:14,841 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/6b3ab89cfbdb472c8cf32a1b586588d0 is 175, key is test_row_0/A:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:14,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742399_1575 (size=31651) 2024-11-17T03:17:14,848 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/bdc57b31a5b843588ac0a6c4eddc4c66 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/bdc57b31a5b843588ac0a6c4eddc4c66 2024-11-17T03:17:14,852 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/B of 97cf29522eb018ba919d51f502de21d3 into bdc57b31a5b843588ac0a6c4eddc4c66(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:14,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:14,852 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/B, priority=10, startTime=1731813434419; duration=0sec 2024-11-17T03:17:14,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T03:17:14,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:B 2024-11-17T03:17:14,852 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T03:17:14,853 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73248 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T03:17:14,853 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1541): 97cf29522eb018ba919d51f502de21d3/C is initiating minor compaction (all files) 2024-11-17T03:17:14,853 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 97cf29522eb018ba919d51f502de21d3/C in TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:14,853 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/516d0ae050bd414e89f7085abfb8bd0b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fa440fe41227404cbe6e0a3f5ea3f224, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/c930fa71db0d445eadc8febd0716196a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/3adb104739fb467886492db2043ab857, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38ae3abacf084499ab1ee61a6c1403d8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38c272bbce8746e6b33296693f98abd0] into tmpdir=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp, totalSize=71.5 K 2024-11-17T03:17:14,854 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 516d0ae050bd414e89f7085abfb8bd0b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731813423802 2024-11-17T03:17:14,854 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting fa440fe41227404cbe6e0a3f5ea3f224, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731813424518 2024-11-17T03:17:14,854 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting c930fa71db0d445eadc8febd0716196a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731813425704 2024-11-17T03:17:14,854 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 3adb104739fb467886492db2043ab857, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731813427869 2024-11-17T03:17:14,855 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ae3abacf084499ab1ee61a6c1403d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731813428547 2024-11-17T03:17:14,855 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c272bbce8746e6b33296693f98abd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731813430718 2024-11-17T03:17:14,863 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 97cf29522eb018ba919d51f502de21d3#C#compaction#483 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T03:17:14,863 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/1870c3f2073648c8a52fa0e650b67841 is 50, key is test_row_0/C:col10/1731813432748/Put/seqid=0 2024-11-17T03:17:14,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742400_1576 (size=12697) 2024-11-17T03:17:14,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. as already flushing 2024-11-17T03:17:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813494910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:14,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813494911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813494914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813494915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:14,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:14,923 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c2a1f9605e6d4d64b94523738d8dc33f_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c2a1f9605e6d4d64b94523738d8dc33f_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/7438b21e68374d099a3cb6cde472ca79, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/7438b21e68374d099a3cb6cde472ca79 is 175, key is test_row_0/A:col10/1731813432794/Put/seqid=0 2024-11-17T03:17:14,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742401_1577 (size=31105) 2024-11-17T03:17:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54954 deadline: 1731813494977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:14,983 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 695c90f4bb18,38753,1731813266381 for 'test_row_2' in 97cf29522eb018ba919d51f502de21d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T03:17:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813495027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813495027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813495027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813495027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813495237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813495237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813495237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813495237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,249 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/6b3ab89cfbdb472c8cf32a1b586588d0 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/6b3ab89cfbdb472c8cf32a1b586588d0 2024-11-17T03:17:15,252 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/A of 97cf29522eb018ba919d51f502de21d3 into 6b3ab89cfbdb472c8cf32a1b586588d0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:15,252 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:15,253 INFO [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/A, priority=10, startTime=1731813434419; duration=0sec 2024-11-17T03:17:15,253 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:15,253 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:A 2024-11-17T03:17:15,273 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/1870c3f2073648c8a52fa0e650b67841 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/1870c3f2073648c8a52fa0e650b67841 2024-11-17T03:17:15,280 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 97cf29522eb018ba919d51f502de21d3/C of 97cf29522eb018ba919d51f502de21d3 into 1870c3f2073648c8a52fa0e650b67841(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T03:17:15,280 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:15,280 INFO [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., storeName=97cf29522eb018ba919d51f502de21d3/C, priority=10, startTime=1731813434419; duration=0sec 2024-11-17T03:17:15,280 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:15,280 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:C 2024-11-17T03:17:15,327 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/7438b21e68374d099a3cb6cde472ca79 2024-11-17T03:17:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/66f3ef18f24c4a709e2156c333cc7b09 is 50, key is test_row_0/B:col10/1731813432794/Put/seqid=0 2024-11-17T03:17:15,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742402_1578 (size=12151) 2024-11-17T03:17:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813495547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813495547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813495547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813495547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:15,741 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/66f3ef18f24c4a709e2156c333cc7b09 2024-11-17T03:17:15,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/d132112e47264e80b030c0bb979c42f8 is 50, key is test_row_0/C:col10/1731813432794/Put/seqid=0 2024-11-17T03:17:15,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742403_1579 (size=12151) 2024-11-17T03:17:16,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54966 deadline: 1731813496057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54926 deadline: 1731813496057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54992 deadline: 1731813496057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:16,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T03:17:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54938 deadline: 1731813496057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 , the old value is region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T03:17:16,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., hostname=695c90f4bb18,38753,1731813266381, seqNum=5 because the exception is null or not the one we care about 2024-11-17T03:17:16,154 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/d132112e47264e80b030c0bb979c42f8 2024-11-17T03:17:16,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/7438b21e68374d099a3cb6cde472ca79 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/7438b21e68374d099a3cb6cde472ca79 2024-11-17T03:17:16,161 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/7438b21e68374d099a3cb6cde472ca79, entries=150, sequenceid=247, filesize=30.4 K 2024-11-17T03:17:16,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/66f3ef18f24c4a709e2156c333cc7b09 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/66f3ef18f24c4a709e2156c333cc7b09 2024-11-17T03:17:16,165 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/66f3ef18f24c4a709e2156c333cc7b09, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T03:17:16,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/d132112e47264e80b030c0bb979c42f8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/d132112e47264e80b030c0bb979c42f8 2024-11-17T03:17:16,170 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/d132112e47264e80b030c0bb979c42f8, entries=150, sequenceid=247, filesize=11.9 K 2024-11-17T03:17:16,172 INFO [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 97cf29522eb018ba919d51f502de21d3 in 1660ms, sequenceid=247, compaction requested=false 2024-11-17T03:17:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:16,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/695c90f4bb18:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T03:17:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T03:17:16,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T03:17:16,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4950 sec 2024-11-17T03:17:16,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.5000 sec 2024-11-17T03:17:16,748 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-11-17T03:17:16,748 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:16,748 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,748 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:16,750 INFO [Thread-2286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2286. 2024-11-17T03:17:16,750 DEBUG [Thread-2286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:16,750 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,750 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:16,751 INFO [Thread-2288 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2288. 2024-11-17T03:17:16,751 DEBUG [Thread-2288 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:16,751 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:16,751 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,752 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-11-17T03:17:16,752 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:16,752 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,752 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:16,752 INFO [Thread-2290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2290. 2024-11-17T03:17:16,752 DEBUG [Thread-2290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:16,752 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,752 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:16,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T03:17:16,837 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38753 {}] regionserver.HRegion(8855): Flush requested on 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:16,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T03:17:17,000 INFO [Thread-2275 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2275. 2024-11-17T03:17:17,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:17,000 DEBUG [Thread-2275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:17,000 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:17,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:17,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:17,000 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:17,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:17,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:17,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174bff1bede4d547bd9ef5e250c87dee2a_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813434913/Put/seqid=0 2024-11-17T03:17:17,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742404_1580 (size=12454) 2024-11-17T03:17:17,070 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-17T03:17:17,070 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-11-17T03:17:17,071 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:17,071 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:17,071 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,071 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,071 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,071 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:17,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:17,079 INFO [Thread-2277 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2277. 2024-11-17T03:17:17,079 DEBUG [Thread-2277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:17,079 DEBUG [Thread-2277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,080 INFO [Thread-2279 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2279. 2024-11-17T03:17:17,080 DEBUG [Thread-2277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,080 DEBUG [Thread-2279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T03:17:17,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:17,080 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,080 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T03:17:17,080 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2870 2024-11-17T03:17:17,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2839 2024-11-17T03:17:17,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2846 2024-11-17T03:17:17,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2857 2024-11-17T03:17:17,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2833 2024-11-17T03:17:17,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T03:17:17,081 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:17:17,081 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:17:17,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:17,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:17,082 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T03:17:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:17,085 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813437084"}]},"ts":"1731813437084"} 2024-11-17T03:17:17,086 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T03:17:17,086 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T03:17:17,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T03:17:17,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, UNASSIGN}] 2024-11-17T03:17:17,088 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, UNASSIGN 2024-11-17T03:17:17,088 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=CLOSING, regionLocation=695c90f4bb18,38753,1731813266381 2024-11-17T03:17:17,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, UNASSIGN because future has completed 2024-11-17T03:17:17,090 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T03:17:17,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381}] 2024-11-17T03:17:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:17,242 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:17,242 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T03:17:17,242 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 97cf29522eb018ba919d51f502de21d3, disabling compactions & flushes 2024-11-17T03:17:17,242 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:17,414 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:17,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174bff1bede4d547bd9ef5e250c87dee2a_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174bff1bede4d547bd9ef5e250c87dee2a_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:17,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/1392c811288a49408b667cc4dc3244b2, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:17,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/1392c811288a49408b667cc4dc3244b2 is 175, key is test_row_0/A:col10/1731813434913/Put/seqid=0 2024-11-17T03:17:17,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742405_1581 (size=31255) 2024-11-17T03:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:17,827 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/1392c811288a49408b667cc4dc3244b2 2024-11-17T03:17:17,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/a77f1e69a25d43e99c9a0e5b156964b1 is 50, key is test_row_0/B:col10/1731813434913/Put/seqid=0 2024-11-17T03:17:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742406_1582 (size=12301) 2024-11-17T03:17:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:18,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/a77f1e69a25d43e99c9a0e5b156964b1 2024-11-17T03:17:18,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/92b8d09b458f4b89a731c1a1511cc9f3 is 50, key is test_row_0/C:col10/1731813434913/Put/seqid=0 2024-11-17T03:17:18,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742407_1583 (size=12301) 2024-11-17T03:17:18,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/92b8d09b458f4b89a731c1a1511cc9f3 2024-11-17T03:17:18,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/1392c811288a49408b667cc4dc3244b2 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1392c811288a49408b667cc4dc3244b2 2024-11-17T03:17:18,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1392c811288a49408b667cc4dc3244b2, entries=150, sequenceid=269, filesize=30.5 K 2024-11-17T03:17:18,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/a77f1e69a25d43e99c9a0e5b156964b1 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/a77f1e69a25d43e99c9a0e5b156964b1 2024-11-17T03:17:18,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/a77f1e69a25d43e99c9a0e5b156964b1, entries=150, sequenceid=269, filesize=12.0 K 2024-11-17T03:17:18,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/92b8d09b458f4b89a731c1a1511cc9f3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/92b8d09b458f4b89a731c1a1511cc9f3 2024-11-17T03:17:18,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/92b8d09b458f4b89a731c1a1511cc9f3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-17T03:17:18,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for 97cf29522eb018ba919d51f502de21d3 in 1678ms, sequenceid=269, compaction requested=true 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 97cf29522eb018ba919d51f502de21d3: 2024-11-17T03:17:18,677 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T03:17:18,677 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:18,677 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. because compaction request was cancelled 2024-11-17T03:17:18,677 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. after waiting 0 ms 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:17:18,677 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:18,677 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:A 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:18,677 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. because compaction request was cancelled 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 97cf29522eb018ba919d51f502de21d3:C, priority=-2147483648, current under compaction store size is 2 2024-11-17T03:17:18,677 DEBUG [RS:0;695c90f4bb18:38753-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:B 2024-11-17T03:17:18,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T03:17:18,677 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. because compaction request was cancelled 2024-11-17T03:17:18,677 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 97cf29522eb018ba919d51f502de21d3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T03:17:18,677 DEBUG [RS:0;695c90f4bb18:38753-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 97cf29522eb018ba919d51f502de21d3:C 2024-11-17T03:17:18,677 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=A 2024-11-17T03:17:18,678 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:18,678 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=B 2024-11-17T03:17:18,678 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:18,678 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 97cf29522eb018ba919d51f502de21d3, store=C 2024-11-17T03:17:18,678 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T03:17:18,683 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111725485be46e034cedbce56174074ac896_97cf29522eb018ba919d51f502de21d3 is 50, key is test_row_0/A:col10/1731813437068/Put/seqid=0 2024-11-17T03:17:18,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742408_1584 (size=12454) 2024-11-17T03:17:19,089 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T03:17:19,097 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111725485be46e034cedbce56174074ac896_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111725485be46e034cedbce56174074ac896_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:19,098 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/710d88b31b724285829a9ed8910d47d7, store: [table=TestAcidGuarantees family=A region=97cf29522eb018ba919d51f502de21d3] 2024-11-17T03:17:19,099 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/710d88b31b724285829a9ed8910d47d7 is 175, key is test_row_0/A:col10/1731813437068/Put/seqid=0 2024-11-17T03:17:19,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742409_1585 (size=31255) 2024-11-17T03:17:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:19,504 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/710d88b31b724285829a9ed8910d47d7 2024-11-17T03:17:19,511 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/dc96c6678b5b46bb943cf160c974dcb8 is 50, key is test_row_0/B:col10/1731813437068/Put/seqid=0 2024-11-17T03:17:19,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742410_1586 (size=12301) 2024-11-17T03:17:19,916 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/dc96c6678b5b46bb943cf160c974dcb8 2024-11-17T03:17:19,925 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/08a0b4e7720649718621a503556df173 is 50, key is test_row_0/C:col10/1731813437068/Put/seqid=0 2024-11-17T03:17:19,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742411_1587 (size=12301) 2024-11-17T03:17:20,329 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/08a0b4e7720649718621a503556df173 2024-11-17T03:17:20,332 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/A/710d88b31b724285829a9ed8910d47d7 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/710d88b31b724285829a9ed8910d47d7 2024-11-17T03:17:20,335 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/710d88b31b724285829a9ed8910d47d7, entries=150, sequenceid=276, filesize=30.5 K 2024-11-17T03:17:20,336 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/B/dc96c6678b5b46bb943cf160c974dcb8 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/dc96c6678b5b46bb943cf160c974dcb8 2024-11-17T03:17:20,339 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/dc96c6678b5b46bb943cf160c974dcb8, entries=150, sequenceid=276, filesize=12.0 K 2024-11-17T03:17:20,339 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/.tmp/C/08a0b4e7720649718621a503556df173 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/08a0b4e7720649718621a503556df173 2024-11-17T03:17:20,342 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/08a0b4e7720649718621a503556df173, entries=150, sequenceid=276, filesize=12.0 K 2024-11-17T03:17:20,343 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 97cf29522eb018ba919d51f502de21d3 in 1666ms, sequenceid=276, compaction requested=true 2024-11-17T03:17:20,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/42acea342955405d8b52e01361302704, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b7945c51b8a64f6e8870f3aa5b7025ae, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1641985a8cb04503a9237dcff3cdb85d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b] to archive 2024-11-17T03:17:20,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:17:20,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/5b00e60e291a44758ab0bd14d69ce843 2024-11-17T03:17:20,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9b0f0b4f36741c0bf093dd2acbf5355 2024-11-17T03:17:20,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/42acea342955405d8b52e01361302704 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/42acea342955405d8b52e01361302704 2024-11-17T03:17:20,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/e9e2a796f99346b09b54abfa2c8b6e7b 2024-11-17T03:17:20,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b098d2b4bf764065a5ec296ee611e7d8 2024-11-17T03:17:20,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b136bf18ff0640529f02112c257e649b 2024-11-17T03:17:20,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b7945c51b8a64f6e8870f3aa5b7025ae to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b7945c51b8a64f6e8870f3aa5b7025ae 2024-11-17T03:17:20,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/15bf0bddf2f54a42b1f765d049e087a9 2024-11-17T03:17:20,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/2422eab2a78544fa9970127a8bd5f85e 2024-11-17T03:17:20,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1641985a8cb04503a9237dcff3cdb85d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1641985a8cb04503a9237dcff3cdb85d 2024-11-17T03:17:20,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/d298a522264d4334b75e0e7e0698da8d 2024-11-17T03:17:20,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/b80ff718239e4d5487ed9a75a36c0bb5 2024-11-17T03:17:20,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/dfbc13f8da244e1db6a27eb361842a96 2024-11-17T03:17:20,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/bf6603d1bc5c4903ad97e0c8b1c35beb 2024-11-17T03:17:20,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/35032aa12d664a748ef4c5cd86daf6da 2024-11-17T03:17:20,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/11f8f419621449b3a48715e33de9a39b 2024-11-17T03:17:20,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59fdc51486a34f38bb2618fbcd81b4f2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26d02481005840eb982945f6fd3bd6d0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/5496daa73af341cba63a101f830fd134, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/1bb82fd5bf9b40039ad53cf73cead1e5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3113ca174f9f4ade81afae090cdf29da, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3a6284c5837e42ac92184d74a8a843b2, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/7ac942427fbb4f09840da0fde1c31ed5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6b1eb8253d314c5fbd84a4c066e12b0d, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26652ee9a36a41668ff59285073f3b92, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/328cf1b2ef4f4382b961ee4e1a3a64b5, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/65d40bbfceac4f289b023f0af8e62c90, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59bd637bcc364f2c9289620be7396e93, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/837babc4dd504ddb8bb3fc9a4d7696c4, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/feb9d3f5aba746cb9a35efe1f0d7357b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3532e78765b14662b23c7b38546e0997, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6ba9d3388f3d461583eb9b40c187e7cd] to archive 2024-11-17T03:17:20,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:17:20,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59fdc51486a34f38bb2618fbcd81b4f2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59fdc51486a34f38bb2618fbcd81b4f2 2024-11-17T03:17:20,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26d02481005840eb982945f6fd3bd6d0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26d02481005840eb982945f6fd3bd6d0 2024-11-17T03:17:20,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/5496daa73af341cba63a101f830fd134 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/5496daa73af341cba63a101f830fd134 2024-11-17T03:17:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/1bb82fd5bf9b40039ad53cf73cead1e5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/1bb82fd5bf9b40039ad53cf73cead1e5 2024-11-17T03:17:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3113ca174f9f4ade81afae090cdf29da to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3113ca174f9f4ade81afae090cdf29da 2024-11-17T03:17:20,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3a6284c5837e42ac92184d74a8a843b2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3a6284c5837e42ac92184d74a8a843b2 2024-11-17T03:17:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/7ac942427fbb4f09840da0fde1c31ed5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/7ac942427fbb4f09840da0fde1c31ed5 2024-11-17T03:17:20,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6b1eb8253d314c5fbd84a4c066e12b0d to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6b1eb8253d314c5fbd84a4c066e12b0d 2024-11-17T03:17:20,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26652ee9a36a41668ff59285073f3b92 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/26652ee9a36a41668ff59285073f3b92 2024-11-17T03:17:20,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/328cf1b2ef4f4382b961ee4e1a3a64b5 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/328cf1b2ef4f4382b961ee4e1a3a64b5 2024-11-17T03:17:20,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/65d40bbfceac4f289b023f0af8e62c90 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/65d40bbfceac4f289b023f0af8e62c90 2024-11-17T03:17:20,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59bd637bcc364f2c9289620be7396e93 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/59bd637bcc364f2c9289620be7396e93 2024-11-17T03:17:20,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/837babc4dd504ddb8bb3fc9a4d7696c4 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/837babc4dd504ddb8bb3fc9a4d7696c4 2024-11-17T03:17:20,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/feb9d3f5aba746cb9a35efe1f0d7357b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/feb9d3f5aba746cb9a35efe1f0d7357b 2024-11-17T03:17:20,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3532e78765b14662b23c7b38546e0997 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/3532e78765b14662b23c7b38546e0997 2024-11-17T03:17:20,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6ba9d3388f3d461583eb9b40c187e7cd to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/6ba9d3388f3d461583eb9b40c187e7cd 2024-11-17T03:17:20,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/780d37160043418e90ed5c257ce172c0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fd297e57bc0c4ba3bc11bdd6743ce8ff, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/9a41af74a92c47318b897e065cdd61e8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7d92841fe59a488580308b3c20492f18, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/b2843dcf567141569635ddf6c89834bb, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7735722f0fee41e2aac1daf5e1bd110a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/afbb4b22da7b4e2190f16835265a85b0, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/db460886786747948e74979aad1af25b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/4a52e07e556344ff94716eb7fb9be4f1, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/516d0ae050bd414e89f7085abfb8bd0b, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/6eb7c4ccac73490a8f2500deb3b0c850, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fa440fe41227404cbe6e0a3f5ea3f224, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/c930fa71db0d445eadc8febd0716196a, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/3adb104739fb467886492db2043ab857, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38ae3abacf084499ab1ee61a6c1403d8, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38c272bbce8746e6b33296693f98abd0] to archive 2024-11-17T03:17:20,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T03:17:20,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/780d37160043418e90ed5c257ce172c0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/780d37160043418e90ed5c257ce172c0 2024-11-17T03:17:20,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fd297e57bc0c4ba3bc11bdd6743ce8ff to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fd297e57bc0c4ba3bc11bdd6743ce8ff 2024-11-17T03:17:20,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/9a41af74a92c47318b897e065cdd61e8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/9a41af74a92c47318b897e065cdd61e8 2024-11-17T03:17:20,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7d92841fe59a488580308b3c20492f18 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7d92841fe59a488580308b3c20492f18 2024-11-17T03:17:20,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/b2843dcf567141569635ddf6c89834bb to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/b2843dcf567141569635ddf6c89834bb 2024-11-17T03:17:20,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7735722f0fee41e2aac1daf5e1bd110a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/7735722f0fee41e2aac1daf5e1bd110a 2024-11-17T03:17:20,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/afbb4b22da7b4e2190f16835265a85b0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/afbb4b22da7b4e2190f16835265a85b0 2024-11-17T03:17:20,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/db460886786747948e74979aad1af25b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/db460886786747948e74979aad1af25b 2024-11-17T03:17:20,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/4a52e07e556344ff94716eb7fb9be4f1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/4a52e07e556344ff94716eb7fb9be4f1 2024-11-17T03:17:20,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/516d0ae050bd414e89f7085abfb8bd0b to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/516d0ae050bd414e89f7085abfb8bd0b 2024-11-17T03:17:20,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/6eb7c4ccac73490a8f2500deb3b0c850 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/6eb7c4ccac73490a8f2500deb3b0c850 2024-11-17T03:17:20,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fa440fe41227404cbe6e0a3f5ea3f224 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/fa440fe41227404cbe6e0a3f5ea3f224 2024-11-17T03:17:20,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/c930fa71db0d445eadc8febd0716196a to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/c930fa71db0d445eadc8febd0716196a 2024-11-17T03:17:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/3adb104739fb467886492db2043ab857 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/3adb104739fb467886492db2043ab857 2024-11-17T03:17:20,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38ae3abacf084499ab1ee61a6c1403d8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38ae3abacf084499ab1ee61a6c1403d8 2024-11-17T03:17:20,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38c272bbce8746e6b33296693f98abd0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/38c272bbce8746e6b33296693f98abd0 2024-11-17T03:17:20,390 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/recovered.edits/279.seqid, newMaxSeqId=279, maxSeqId=4 2024-11-17T03:17:20,390 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. 2024-11-17T03:17:20,391 DEBUG [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 97cf29522eb018ba919d51f502de21d3: Waiting for close lock at 1731813437242Running coprocessor pre-close hooks at 1731813437242Disabling compacts and flushes for region at 1731813437242Disabling writes for close at 1731813438677 (+1435 ms)Obtaining lock to block concurrent updates at 1731813438677Preparing flush snapshotting stores in 97cf29522eb018ba919d51f502de21d3 at 1731813438677Finished memstore snapshotting TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731813438678 (+1 ms)Flushing stores of TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3. at 1731813438678Flushing 97cf29522eb018ba919d51f502de21d3/A: creating writer at 1731813438678Flushing mob file 97cf29522eb018ba919d51f502de21d3/A: appending metadata at 1731813438683 (+5 ms)Flushing mob file 97cf29522eb018ba919d51f502de21d3/A: closing flushed file at 1731813438683Flushing 97cf29522eb018ba919d51f502de21d3/A: appending metadata at 1731813439098 (+415 ms)Flushing 97cf29522eb018ba919d51f502de21d3/A: closing flushed file at 1731813439099 (+1 ms)Flushing 97cf29522eb018ba919d51f502de21d3/B: creating writer at 1731813439508 (+409 ms)Flushing 97cf29522eb018ba919d51f502de21d3/B: appending metadata at 1731813439510 (+2 ms)Flushing 97cf29522eb018ba919d51f502de21d3/B: closing flushed file at 1731813439510Flushing 97cf29522eb018ba919d51f502de21d3/C: creating writer at 1731813439922 (+412 ms)Flushing 97cf29522eb018ba919d51f502de21d3/C: appending metadata at 1731813439924 (+2 ms)Flushing 97cf29522eb018ba919d51f502de21d3/C: closing flushed file at 1731813439924Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4660dda9: reopening flushed file at 1731813440331 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1280af0: reopening flushed file at 1731813440335 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d3ae480: reopening flushed file at 1731813440339 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 97cf29522eb018ba919d51f502de21d3 in 1666ms, sequenceid=276, compaction requested=true at 1731813440343 (+4 ms)Writing region close event to WAL at 1731813440388 (+45 ms)Running coprocessor post-close hooks at 1731813440390 (+2 ms)Closed at 1731813440390 2024-11-17T03:17:20,392 INFO [RS_CLOSE_REGION-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:20,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=97cf29522eb018ba919d51f502de21d3, regionState=CLOSED 2024-11-17T03:17:20,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 because future has completed 2024-11-17T03:17:20,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T03:17:20,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 97cf29522eb018ba919d51f502de21d3, server=695c90f4bb18,38753,1731813266381 in 3.3040 sec 2024-11-17T03:17:20,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-17T03:17:20,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=97cf29522eb018ba919d51f502de21d3, UNASSIGN in 3.3080 sec 2024-11-17T03:17:20,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T03:17:20,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3100 sec 2024-11-17T03:17:20,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731813440398"}]},"ts":"1731813440398"} 2024-11-17T03:17:20,399 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T03:17:20,400 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T03:17:20,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3180 sec 2024-11-17T03:17:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T03:17:21,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:21,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T03:17:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:21,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T03:17:21,244 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:21,248 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,252 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C, FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/recovered.edits] 2024-11-17T03:17:21,255 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1392c811288a49408b667cc4dc3244b2 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/1392c811288a49408b667cc4dc3244b2 2024-11-17T03:17:21,257 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/6b3ab89cfbdb472c8cf32a1b586588d0 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/6b3ab89cfbdb472c8cf32a1b586588d0 2024-11-17T03:17:21,258 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/710d88b31b724285829a9ed8910d47d7 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/710d88b31b724285829a9ed8910d47d7 2024-11-17T03:17:21,259 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/7438b21e68374d099a3cb6cde472ca79 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/A/7438b21e68374d099a3cb6cde472ca79 2024-11-17T03:17:21,261 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/66f3ef18f24c4a709e2156c333cc7b09 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/66f3ef18f24c4a709e2156c333cc7b09 2024-11-17T03:17:21,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/a77f1e69a25d43e99c9a0e5b156964b1 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/a77f1e69a25d43e99c9a0e5b156964b1 2024-11-17T03:17:21,263 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/bdc57b31a5b843588ac0a6c4eddc4c66 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/bdc57b31a5b843588ac0a6c4eddc4c66 2024-11-17T03:17:21,264 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/dc96c6678b5b46bb943cf160c974dcb8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/B/dc96c6678b5b46bb943cf160c974dcb8 2024-11-17T03:17:21,266 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/08a0b4e7720649718621a503556df173 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/08a0b4e7720649718621a503556df173 2024-11-17T03:17:21,268 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/1870c3f2073648c8a52fa0e650b67841 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/1870c3f2073648c8a52fa0e650b67841 2024-11-17T03:17:21,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/92b8d09b458f4b89a731c1a1511cc9f3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/92b8d09b458f4b89a731c1a1511cc9f3 2024-11-17T03:17:21,271 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/d132112e47264e80b030c0bb979c42f8 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/C/d132112e47264e80b030c0bb979c42f8 2024-11-17T03:17:21,273 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/recovered.edits/279.seqid to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3/recovered.edits/279.seqid 2024-11-17T03:17:21,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/default/TestAcidGuarantees/97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,274 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T03:17:21,274 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T03:17:21,275 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T03:17:21,277 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111708de94844bda41029cc20cfac465a936_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111708de94844bda41029cc20cfac465a936_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,278 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171718c5de53f244468185b5dde4e3d8b2_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171718c5de53f244468185b5dde4e3d8b2_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,279 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111725485be46e034cedbce56174074ac896_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111725485be46e034cedbce56174074ac896_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,280 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e43d4c729cf4ebb92e870361a864ce1_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e43d4c729cf4ebb92e870361a864ce1_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,281 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e78e600d3cd44538cdff2a2c5dd1e86_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e78e600d3cd44538cdff2a2c5dd1e86_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,282 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174bff1bede4d547bd9ef5e250c87dee2a_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174bff1bede4d547bd9ef5e250c87dee2a_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,282 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174da056788c8045a0837129717754118f_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174da056788c8045a0837129717754118f_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,283 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b586fb2e553842388c356397b6dc33e4_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b586fb2e553842388c356397b6dc33e4_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,284 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c2a1f9605e6d4d64b94523738d8dc33f_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c2a1f9605e6d4d64b94523738d8dc33f_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,285 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d0376879fc744216bc5a1dafdf53216f_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d0376879fc744216bc5a1dafdf53216f_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,286 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e2cd212a569441b28af24a4513ba43b0_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e2cd212a569441b28af24a4513ba43b0_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,287 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e46d1adad1fd42eabdd0439c53fc9805_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e46d1adad1fd42eabdd0439c53fc9805_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,288 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e7f71be590cb41c282b6fbe59a60d37b_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e7f71be590cb41c282b6fbe59a60d37b_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,288 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ed70ff775c244aebb70c2673ce5a1ce3_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ed70ff775c244aebb70c2673ce5a1ce3_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,289 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f4831f90800e48dfae3faca25a1251b1_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f4831f90800e48dfae3faca25a1251b1_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,290 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f651a3572c3a4be89f4ebcfb1a6982fc_97cf29522eb018ba919d51f502de21d3 to hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f651a3572c3a4be89f4ebcfb1a6982fc_97cf29522eb018ba919d51f502de21d3 2024-11-17T03:17:21,290 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T03:17:21,292 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:21,294 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T03:17:21,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T03:17:21,296 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:21,296 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T03:17:21,296 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731813441296"}]},"ts":"9223372036854775807"} 2024-11-17T03:17:21,298 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T03:17:21,298 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 97cf29522eb018ba919d51f502de21d3, NAME => 'TestAcidGuarantees,,1731813413529.97cf29522eb018ba919d51f502de21d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T03:17:21,298 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T03:17:21,298 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731813441298"}]},"ts":"9223372036854775807"} 2024-11-17T03:17:21,299 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T03:17:21,300 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T03:17:21,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-17T03:17:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T03:17:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T03:17:21,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T03:17:21,362 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 462), ProcessCount=11 (was 11), AvailableMemoryMB=1648 (was 1066) - AvailableMemoryMB LEAK? - 2024-11-17T03:17:21,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T03:17:21,362 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:17:21,362 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:17:21,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:21,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:21,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T03:17:21,363 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T03:17:21,363 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-17T03:17:21,363 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=695c90f4bb18,42091,1731813265296 2024-11-17T03:17:21,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T03:17:21,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T03:17:21,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:17:21,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:17:21,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T03:17:21,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T03:17:21,430 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:17:21,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:21,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T03:17:21,431 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T03:17:21,431 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '695c90f4bb18,38753,1731813266381' ***** 2024-11-17T03:17:21,432 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T03:17:21,432 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T03:17:21,433 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(959): stopping server 695c90f4bb18,38753,1731813266381 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;695c90f4bb18:38753. 2024-11-17T03:17:21,433 DEBUG [RS:0;695c90f4bb18:38753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T03:17:21,433 DEBUG [RS:0;695c90f4bb18:38753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T03:17:21,433 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T03:17:21,434 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T03:17:21,434 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T03:17:21,434 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T03:17:21,435 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:21,435 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T03:17:21,435 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T03:17:21,435 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T03:17:21,435 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T03:17:21,435 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T03:17:21,435 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T03:17:21,455 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/info/4c2f32a90b9d478898ac02e55a4c8254 is 95, key is TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77./info:/1731813302797/DeleteFamily/seqid=0 2024-11-17T03:17:21,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742412_1588 (size=6514) 2024-11-17T03:17:21,477 INFO [regionserver/695c90f4bb18:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T03:17:21,606 INFO [regionserver/695c90f4bb18:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T03:17:21,606 INFO [regionserver/695c90f4bb18:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T03:17:21,635 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:21,835 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:21,859 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/info/4c2f32a90b9d478898ac02e55a4c8254 2024-11-17T03:17:21,880 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/ns/3e996d4187ea48f1af3379dc77e5e0ca is 93, key is TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77./ns:/1731813302754/DeleteFamily/seqid=0 2024-11-17T03:17:21,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742413_1589 (size=5906) 2024-11-17T03:17:22,035 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:22,236 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:22,286 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/ns/3e996d4187ea48f1af3379dc77e5e0ca 2024-11-17T03:17:22,311 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/rep_barrier/0653fe7cd71d49969895ffda42583647 is 102, key is TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77./rep_barrier:/1731813302754/DeleteFamily/seqid=0 2024-11-17T03:17:22,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742414_1590 (size=6025) 2024-11-17T03:17:22,436 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:22,636 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:22,719 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/rep_barrier/0653fe7cd71d49969895ffda42583647 2024-11-17T03:17:22,749 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/table/e738f859f6eb4ae3b1514890d36bc0a3 is 96, key is TestAcidGuarantees,,1731813271225.b29e9d27b06f8df48f1da9682bc65b77./table:/1731813302754/DeleteFamily/seqid=0 2024-11-17T03:17:22,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742415_1591 (size=5936) 2024-11-17T03:17:22,836 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:23,037 DEBUG [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T03:17:23,152 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/table/e738f859f6eb4ae3b1514890d36bc0a3 2024-11-17T03:17:23,156 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/info/4c2f32a90b9d478898ac02e55a4c8254 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/info/4c2f32a90b9d478898ac02e55a4c8254 2024-11-17T03:17:23,159 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/info/4c2f32a90b9d478898ac02e55a4c8254, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T03:17:23,160 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/ns/3e996d4187ea48f1af3379dc77e5e0ca as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/ns/3e996d4187ea48f1af3379dc77e5e0ca 2024-11-17T03:17:23,163 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/ns/3e996d4187ea48f1af3379dc77e5e0ca, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T03:17:23,164 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/rep_barrier/0653fe7cd71d49969895ffda42583647 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/rep_barrier/0653fe7cd71d49969895ffda42583647 2024-11-17T03:17:23,167 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/rep_barrier/0653fe7cd71d49969895ffda42583647, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T03:17:23,167 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/.tmp/table/e738f859f6eb4ae3b1514890d36bc0a3 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/table/e738f859f6eb4ae3b1514890d36bc0a3 2024-11-17T03:17:23,171 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/table/e738f859f6eb4ae3b1514890d36bc0a3, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T03:17:23,171 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1736ms, sequenceid=90, compaction requested=false 2024-11-17T03:17:23,179 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T03:17:23,180 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T03:17:23,180 INFO [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T03:17:23,180 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731813441434Running coprocessor pre-close hooks at 1731813441435 (+1 ms)Disabling compacts and flushes for region at 1731813441435Disabling writes for close at 1731813441435Obtaining lock to block concurrent updates at 1731813441435Preparing flush snapshotting stores in 1588230740 at 1731813441435Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731813441436 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731813441436Flushing 1588230740/info: creating writer at 1731813441437 (+1 ms)Flushing 1588230740/info: appending metadata at 1731813441455 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731813441455Flushing 1588230740/ns: creating writer at 1731813441863 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731813441879 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731813441879Flushing 1588230740/rep_barrier: creating writer at 1731813442293 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731813442310 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731813442310Flushing 1588230740/table: creating writer at 1731813442727 (+417 ms)Flushing 1588230740/table: appending metadata at 1731813442749 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731813442749Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f320eab: reopening flushed file at 1731813443155 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@543121f4: reopening flushed file at 1731813443159 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37238b0c: reopening flushed file at 1731813443163 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@754cd9af: reopening flushed file at 1731813443167 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1736ms, sequenceid=90, compaction requested=false at 1731813443171 (+4 ms)Writing region close event to WAL at 1731813443176 (+5 ms)Running coprocessor post-close hooks at 1731813443179 (+3 ms)Closed at 1731813443180 (+1 ms) 2024-11-17T03:17:23,180 DEBUG [RS_CLOSE_META-regionserver/695c90f4bb18:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T03:17:23,237 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(976): stopping server 695c90f4bb18,38753,1731813266381; all regions closed. 2024-11-17T03:17:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741834_1010 (size=25344) 2024-11-17T03:17:23,247 DEBUG [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/oldWALs 2024-11-17T03:17:23,247 INFO [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 695c90f4bb18%2C38753%2C1731813266381.meta:.meta(num 1731813270790) 2024-11-17T03:17:23,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741832_1008 (size=20266510) 2024-11-17T03:17:23,251 DEBUG [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/oldWALs 2024-11-17T03:17:23,251 INFO [RS:0;695c90f4bb18:38753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 695c90f4bb18%2C38753%2C1731813266381:(num 1731813269760) 2024-11-17T03:17:23,251 DEBUG [RS:0;695c90f4bb18:38753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T03:17:23,251 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T03:17:23,251 INFO [RS:0;695c90f4bb18:38753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T03:17:23,252 INFO [RS:0;695c90f4bb18:38753 {}] hbase.ChoreService(370): Chore service for: regionserver/695c90f4bb18:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T03:17:23,252 INFO [RS:0;695c90f4bb18:38753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T03:17:23,252 INFO [regionserver/695c90f4bb18:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T03:17:23,253 INFO [RS:0;695c90f4bb18:38753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38753 2024-11-17T03:17:23,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/695c90f4bb18,38753,1731813266381 2024-11-17T03:17:23,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T03:17:23,323 INFO [RS:0;695c90f4bb18:38753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T03:17:23,344 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [695c90f4bb18,38753,1731813266381] 2024-11-17T03:17:23,364 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/695c90f4bb18,38753,1731813266381 already deleted, retry=false 2024-11-17T03:17:23,364 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 695c90f4bb18,38753,1731813266381 expired; onlineServers=0 2024-11-17T03:17:23,364 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '695c90f4bb18,42091,1731813265296' ***** 2024-11-17T03:17:23,364 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T03:17:23,364 INFO [M:0;695c90f4bb18:42091 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T03:17:23,365 INFO [M:0;695c90f4bb18:42091 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T03:17:23,365 DEBUG [M:0;695c90f4bb18:42091 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T03:17:23,365 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T03:17:23,365 DEBUG [M:0;695c90f4bb18:42091 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T03:17:23,365 DEBUG [master/695c90f4bb18:0:becomeActiveMaster-HFileCleaner.small.0-1731813269435 {}] cleaner.HFileCleaner(306): Exit Thread[master/695c90f4bb18:0:becomeActiveMaster-HFileCleaner.small.0-1731813269435,5,FailOnTimeoutGroup] 2024-11-17T03:17:23,365 DEBUG [master/695c90f4bb18:0:becomeActiveMaster-HFileCleaner.large.0-1731813269434 {}] cleaner.HFileCleaner(306): Exit Thread[master/695c90f4bb18:0:becomeActiveMaster-HFileCleaner.large.0-1731813269434,5,FailOnTimeoutGroup] 2024-11-17T03:17:23,365 INFO [M:0;695c90f4bb18:42091 {}] hbase.ChoreService(370): Chore service for: master/695c90f4bb18:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T03:17:23,365 INFO [M:0;695c90f4bb18:42091 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T03:17:23,365 DEBUG [M:0;695c90f4bb18:42091 {}] master.HMaster(1795): Stopping service threads 2024-11-17T03:17:23,365 INFO [M:0;695c90f4bb18:42091 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T03:17:23,365 INFO [M:0;695c90f4bb18:42091 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T03:17:23,365 ERROR [M:0;695c90f4bb18:42091 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1420519831) connection to localhost/127.0.0.1:39789 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39789,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T03:17:23,366 INFO [M:0;695c90f4bb18:42091 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T03:17:23,366 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T03:17:23,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T03:17:23,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T03:17:23,385 DEBUG [M:0;695c90f4bb18:42091 {}] zookeeper.ZKUtil(347): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T03:17:23,385 WARN [M:0;695c90f4bb18:42091 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T03:17:23,386 INFO [M:0;695c90f4bb18:42091 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/.lastflushedseqids 2024-11-17T03:17:23,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742416_1592 (size=72) 2024-11-17T03:17:23,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T03:17:23,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38753-0x10146c947c80001, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T03:17:23,445 INFO [RS:0;695c90f4bb18:38753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T03:17:23,446 INFO [RS:0;695c90f4bb18:38753 {}] regionserver.HRegionServer(1031): Exiting; stopping=695c90f4bb18,38753,1731813266381; zookeeper connection closed. 2024-11-17T03:17:23,446 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ffe78c3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ffe78c3 2024-11-17T03:17:23,447 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T03:17:23,796 INFO [M:0;695c90f4bb18:42091 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T03:17:23,796 INFO [M:0;695c90f4bb18:42091 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T03:17:23,797 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T03:17:23,797 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:17:23,797 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:17:23,797 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T03:17:23,797 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:17:23,797 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=740.20 KB heapSize=903.66 KB 2024-11-17T03:17:23,822 DEBUG [M:0;695c90f4bb18:42091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c34c002141f54c0e8a6563d818458be9 is 82, key is hbase:meta,,1/info:regioninfo/1731813270871/Put/seqid=0 2024-11-17T03:17:23,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742417_1593 (size=5672) 2024-11-17T03:17:24,249 INFO [M:0;695c90f4bb18:42091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2008 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c34c002141f54c0e8a6563d818458be9 2024-11-17T03:17:24,276 DEBUG [M:0;695c90f4bb18:42091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/acd7bed0334741ae9a17c977ca7fce65 is 2480, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1731813305979/Put/seqid=0 2024-11-17T03:17:24,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742418_1594 (size=44723) 2024-11-17T03:17:24,353 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T03:17:24,680 INFO [M:0;695c90f4bb18:42091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=739.64 KB at sequenceid=2008 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/acd7bed0334741ae9a17c977ca7fce65 2024-11-17T03:17:24,687 INFO [M:0;695c90f4bb18:42091 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acd7bed0334741ae9a17c977ca7fce65 2024-11-17T03:17:24,708 DEBUG [M:0;695c90f4bb18:42091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33e048518dc24f0ea3c2674cb79a761f is 69, key is 695c90f4bb18,38753,1731813266381/rs:state/1731813269523/Put/seqid=0 2024-11-17T03:17:24,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073742419_1595 (size=5156) 2024-11-17T03:17:25,112 INFO [M:0;695c90f4bb18:42091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2008 (bloomFilter=true), to=hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33e048518dc24f0ea3c2674cb79a761f 2024-11-17T03:17:25,115 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c34c002141f54c0e8a6563d818458be9 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c34c002141f54c0e8a6563d818458be9 2024-11-17T03:17:25,118 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c34c002141f54c0e8a6563d818458be9, entries=8, sequenceid=2008, filesize=5.5 K 2024-11-17T03:17:25,119 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/acd7bed0334741ae9a17c977ca7fce65 as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/acd7bed0334741ae9a17c977ca7fce65 2024-11-17T03:17:25,121 INFO [M:0;695c90f4bb18:42091 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acd7bed0334741ae9a17c977ca7fce65 2024-11-17T03:17:25,122 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/acd7bed0334741ae9a17c977ca7fce65, entries=166, sequenceid=2008, filesize=43.7 K 2024-11-17T03:17:25,122 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33e048518dc24f0ea3c2674cb79a761f as hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33e048518dc24f0ea3c2674cb79a761f 2024-11-17T03:17:25,124 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39789/user/jenkins/test-data/644ab75c-337a-6091-edbf-9b804e384f64/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33e048518dc24f0ea3c2674cb79a761f, entries=1, sequenceid=2008, filesize=5.0 K 2024-11-17T03:17:25,125 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(3140): Finished flush of dataSize ~740.20 KB/757962, heapSize ~903.37 KB/925048, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1328ms, sequenceid=2008, compaction requested=false 2024-11-17T03:17:25,126 INFO [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T03:17:25,126 DEBUG [M:0;695c90f4bb18:42091 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731813443796Disabling compacts and flushes for region at 1731813443796Disabling writes for close at 1731813443797 (+1 ms)Obtaining lock to block concurrent updates at 1731813443797Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731813443797Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=757962, getHeapSize=925288, getOffHeapSize=0, getCellsCount=2199 at 1731813443797Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731813443798 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731813443798Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731813443822 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731813443822Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731813444255 (+433 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731813444276 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731813444276Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731813444687 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731813444707 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731813444707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4200ba80: reopening flushed file at 1731813445114 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c8e6af6: reopening flushed file at 1731813445118 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c508fa: reopening flushed file at 1731813445122 (+4 ms)Finished flush of dataSize ~740.20 KB/757962, heapSize ~903.37 KB/925048, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1328ms, sequenceid=2008, compaction requested=false at 1731813445125 (+3 ms)Writing region close event to WAL at 1731813445126 (+1 ms)Closed at 1731813445126 2024-11-17T03:17:25,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741830_1006 (size=889665) 2024-11-17T03:17:25,129 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T03:17:25,129 INFO [M:0;695c90f4bb18:42091 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T03:17:25,129 INFO [M:0;695c90f4bb18:42091 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42091 2024-11-17T03:17:25,129 INFO [M:0;695c90f4bb18:42091 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T03:17:25,243 INFO [M:0;695c90f4bb18:42091 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T03:17:25,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T03:17:25,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42091-0x10146c947c80000, quorum=127.0.0.1:60535, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T03:17:25,252 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T03:17:25,255 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T03:17:25,255 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T03:17:25,255 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T03:17:25,255 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/hadoop.log.dir/,STOPPED} 2024-11-17T03:17:25,258 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T03:17:25,258 WARN [BP-617005350-172.17.0.3-1731813261238 heartbeating to localhost/127.0.0.1:39789 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T03:17:25,259 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T03:17:25,259 WARN [BP-617005350-172.17.0.3-1731813261238 heartbeating to localhost/127.0.0.1:39789 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-617005350-172.17.0.3-1731813261238 (Datanode Uuid 6e400762-c501-488a-ad00-d12d75c2fc6e) service to localhost/127.0.0.1:39789 2024-11-17T03:17:25,261 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/data/data1/current/BP-617005350-172.17.0.3-1731813261238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T03:17:25,262 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/cluster_1b24fd50-aa2c-746f-99ae-012eff19a50d/data/data2/current/BP-617005350-172.17.0.3-1731813261238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T03:17:25,262 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T03:17:25,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T03:17:25,273 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T03:17:25,273 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T03:17:25,273 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T03:17:25,273 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7af55f57-11e1-7910-16ec-d4f86a123acd/hadoop.log.dir/,STOPPED} 2024-11-17T03:17:25,286 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T03:17:25,392 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-17T03:17:25,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down